merge db sources

src/tbd/core.clj

@@ -27,6 +27,7 @@
             [tbd.nrepl.routes :as nrepl-routes]
             [tbd.oauth.db :as oauth-db]
             [tbd.oauth.routes :as oauth-routes]
+            [tbd.sqlite :as sqlite]
             [tbd.todo.db :as todo-db]
             [tbd.todo.routes :as todo-routes]))
 
@@ -70,10 +71,9 @@
 
 ;; Each logical group (health, users, orders, ...) owns a `tbd.<group>.routes`
 ;; namespace that returns its own route subtree; they're concatenated here.
-(defn make-router [{:keys [auth nrepl todo oauth webmcp chat]}]
-  (let [oauth-ds (delay (oauth-db/open (:db-path oauth)))
-        ;; Opened (and migrated) on first use, not when the app loads.
-        todo-ds (delay (todo-db/open (:db-path todo)))]
+(defn make-router [{:keys [auth nrepl db webmcp chat]}]
+  ;; One database for every group's tables, opened (and migrated) on first use, not when the app loads.
+  (let [ds (delay (sqlite/open (:db-path db) todo-db/migrate! oauth-db/migrate!))]
     (ring/router
       [["/swagger.json"
         ;; The spec is JSON, whatever the API default is (Swagger UI can't read EDN).
@@ -85,12 +85,12 @@
                    :handler (index-handler webmcp)}}]
        (health/routes)
        (math/routes)
-       (mcp-routes/routes auth oauth-ds todo-ds)
-       (oauth-routes/routes auth oauth-ds)
+       (mcp-routes/routes auth ds)
+       (oauth-routes/routes auth ds)
        (auth-routes/routes auth)
        (chat-routes/routes chat)
        (nrepl-routes/routes nrepl)
-       (todo-routes/routes todo-ds)]
+       (todo-routes/routes ds)]
       {:data {:muuntaja http/muuntaja
               :coercion malli/coercion
               ;; Handlers (and the auth/error middleware) return plain data; the muuntaja
@@ -124,8 +124,7 @@
   {:auth (auth/env-config)
    :nrepl (nrepl/env-config)
    :chat (chat/env-config)
-   :todo (todo-db/env-config)
-   :oauth (oauth-db/env-config)
+   :db (sqlite/env-config)
    :webmcp {:origin-trial-token (some-> (System/getenv "WEBMCP_ORIGIN_TRIAL_TOKEN") str/trim not-empty)}})
 
 (def app (make-app config))

src/tbd/mcp/routes.clj

@@ -18,9 +18,9 @@
 
 (defn make-handler
   "A Ring handler speaking MCP over Streamable HTTP, for callers with a valid access token
-  (`auth` is the auth config, `ds` a delay of the OAuth database, `todo-ds` one of the todo database). It keeps its MCP sessions in
+  (`auth` is the auth config, `ds` a delay of the app's database). It keeps its MCP sessions in
   memory, so each handler is its own server: sessions don't outlive it, and aren't shared between nodes."
-  [auth ds todo-ds]
+  [auth ds]
   (let [{:keys [runtime jsonrpc-handler] :as options}
         (mcp-server/make-server-options
           {:info (es/make-info "tbd" "1.0.0" "tbd MCP server")
@@ -29,14 +29,14 @@
            :traffic-logger traffic-logger/nop-traffic-logger})]
     (-> (mcp-http/make-ring-handler runtime jsonrpc-handler options)
         ;; What the tools that touch todos need besides the caller (see tbd.mcp.tools).
-        (wrap-runtime {tools/todo-ds-key todo-ds})
+        (wrap-runtime {tools/ds-key ds})
         ;; Jetty hands over the body as a stream; plumcp reads the message through `:on-msg`.
         mcp-http/wrap-request-body-reader
         ;; Outermost, so a request without a token is turned away before its body is read.
         (oauth-mw/wrap-bearer-auth auth ds))))
 
-(defn routes [auth ds todo-ds]
-  (let [handler (make-handler auth ds todo-ds)]
+(defn routes [auth ds]
+  (let [handler (make-handler auth ds)]
     ["/mcp"
      ;; plumcp does its own content negotiation, JSON parsing and (via the wrapper above) body
      ;; reading, so it must not sit behind the API's muuntaja/coercion middleware chain. ^:replace

src/tbd/mcp/tools.clj

@@ -18,10 +18,10 @@
             [tbd.todo.schema :as todo-schema]
             [tbd.todo.service :as todo]))
 
-(def todo-ds-key
-  "Where the todo database (a delay, as for the HTTP routes) is in the request's plumcp runtime
-  bag; tbd.mcp.routes puts it there."
-  :tbd/todo-ds)
+(def ds-key
+  "Where the database (a delay, as for the HTTP routes) is in the request's plumcp runtime bag;
+  tbd.mcp.routes puts it there."
+  :tbd/ds)
 
 (defn- current-user
   "Who is calling: {:login :id :admin? :client-id}, the user whose access token the request
@@ -39,7 +39,7 @@
   (if error (throw (ex-info error {})) ok))
 
 ;; Todos belong to the caller, and tbd.todo.service scopes every operation to them.
-(defn- todo-ds [ctx] (ctx-get ctx todo-ds-key))
+(defn- ds [ctx] (ctx-get ctx ds-key))
 
 (def primitives
   {:tools
@@ -55,20 +55,20 @@
               (format "%s (Gitea user %d)%s" login id (if admin? ", admin" "")))))
 
     (tool "list_todos" "Lists your todos (JSON: id, title, done, created-at, updated-at), oldest first" [:map]
-          (fn [_ ctx] (->json (ok! (todo/list-todos (todo-ds ctx) (current-user ctx))))))
+          (fn [_ ctx] (->json (ok! (todo/list-todos (ds ctx) (current-user ctx))))))
 
     (tool "get_todo" "Gets one of your todos by id (JSON)" todo-schema/IdParam
-          (fn [{:keys [id]} ctx] (->json (ok! (todo/find-todo (todo-ds ctx) (current-user ctx) id)))))
+          (fn [{:keys [id]} ctx] (->json (ok! (todo/find-todo (ds ctx) (current-user ctx) id)))))
 
     (tool "create_todo" "Creates a todo for you and returns it (JSON)" todo-schema/NewTodo
-          (fn [{:keys [title]} ctx] (->json (ok! (todo/create! (todo-ds ctx) (current-user ctx) title)))))
+          (fn [{:keys [title]} ctx] (->json (ok! (todo/create! (ds ctx) (current-user ctx) title)))))
 
     (tool "update_todo" "Changes the title and/or done flag of one of your todos and returns it (JSON)"
           todo-schema/UpdateTodo
           (fn [{:keys [id] :as input} ctx]
-            (->json (ok! (todo/update! (todo-ds ctx) (current-user ctx) id (dissoc input :id))))))
+            (->json (ok! (todo/update! (ds ctx) (current-user ctx) id (dissoc input :id))))))
 
     (tool "delete_todo" "Deletes one of your todos" todo-schema/IdParam
           (fn [{:keys [id]} ctx]
-            (ok! (todo/delete! (todo-ds ctx) (current-user ctx) id))
+            (ok! (todo/delete! (ds ctx) (current-user ctx) id))
             "deleted"))]})

src/tbd/oauth/db.clj

@@ -1,23 +1,12 @@
 (ns tbd.oauth.db
-  "SQLite persistence for the MCP OAuth server, in a local file. Storage only: what is valid, and
+  "SQLite persistence for the MCP OAuth server, in the app's database (see tbd.sqlite). Storage only: what is valid, and
   when, lives in tbd.oauth.core and tbd.oauth.routes. Codes and tokens are stored as hashes
   (see tbd.oauth.core/hash-token), so callers pass hashes here, never the secrets themselves."
   (:require [clojure.edn :as edn]
-            [clojure.java.io :as io]
-            [clojure.string :as str]
             [next.jdbc :as jdbc]
             [next.jdbc.result-set :as rs]
             [tbd.sqlite :as sqlite]))
 
-(def default-path "data/oauth.db")
-
-(defn env-config
-  "OAUTH_DB_PATH: where the SQLite file lives (default data/oauth.db; its directory is created)."
-  ([] (env-config (System/getenv)))
-  ([env]
-   (let [path (get env "OAUTH_DB_PATH")]
-     {:db-path (if (str/blank? path) default-path (str/trim path))})))
-
 ;; A user's approval of a client: the current access and refresh token (hashed), replaced
 ;; together when the refresh token is used. Deleting the row revokes the approval. `id` is a
 ;; UUID (see tbd.sqlite/new-id); `user_id` is the user's id in Gitea, which the app doesn't issue.
@@ -34,7 +23,9 @@
      refresh_expires_at INTEGER NOT NULL,
      created_at         INTEGER NOT NULL)")
 
-(defn- migrate! [ds]
+(defn migrate!
+  "Creates the OAuth tables in the database `ds`, or brings them up to date."
+  [ds]
   ;; Clients register themselves (RFC 7591), so a row here says nothing about who the user is.
   (jdbc/execute! ds ["CREATE TABLE IF NOT EXISTS clients (
                         id            TEXT    PRIMARY KEY,
@@ -56,15 +47,10 @@
   (jdbc/execute! ds ["CREATE INDEX IF NOT EXISTS grants_by_client ON grants (client_id)"]))
 
 (defn open
-  "A datasource for the database file at `path`, created and migrated if need be; connections
-  and locking work as in tbd.todo.db."
+  "A database with just the OAuth tables, at `path`; the app opens one with every group's tables
+  (see tbd.core), which this is for tests of the group on its own."
   [path]
-  (when (str/blank? path)
-    (throw (ex-info "no OAuth database path configured (set OAUTH_DB_PATH)" {})))
-  (some-> (io/file path) .getAbsoluteFile .getParentFile .mkdirs)
-  (doto (jdbc/get-datasource {:dbtype "sqlite" :dbname path
-                              :journal_mode "WAL" :busy_timeout 5000})
-    migrate!))
+  (sqlite/open path migrate!))
 
 (def ^:private opts {:builder-fn rs/as-unqualified-kebab-maps})
 

src/tbd/oauth/routes.clj

@@ -303,7 +303,7 @@
       (token-error "unsupported_grant_type" "grant_type must be authorization_code or refresh_token"))))
 
 (defn routes
-  "`ds` is a delay of the datasource (see tbd.oauth.db/open), so the database isn't touched until needed."
+  "`ds` is a delay of the datasource (see tbd.sqlite/open), so the database isn't touched until needed."
   [config ds]
   (let [guarded #(only-when-configured config %)]
     ["" {:no-doc true

src/tbd/sqlite.clj

@@ -1,10 +1,35 @@
 (ns tbd.sqlite
-  "What the app's SQLite databases (tbd.todo.db, tbd.oauth.db) share: the ids of the rows they
-  own, and moving a table off the integer ids it used to have."
-  (:require [clojure.string :as str]
+  "The app's one SQLite database, which every group with storage (tbd.todo.db, tbd.oauth.db)
+  keeps its tables in: where it lives, opening it, the ids of the rows the groups own, and
+  moving a table off the integer ids it used to have."
+  (:require [clojure.java.io :as io]
+            [clojure.string :as str]
             [next.jdbc :as jdbc]
             [next.jdbc.result-set :as rs]))
 
+(def default-path "data/tbd.db")
+
+(defn env-config
+  "DB_PATH: where the SQLite file lives (default data/tbd.db; its directory is created)."
+  ([] (env-config (System/getenv)))
+  ([env]
+   (let [path (get env "DB_PATH")]
+     {:db-path (if (str/blank? path) default-path (str/trim path))})))
+
+(defn open
+  "A datasource for the database file at `path`, created if need be and migrated by each of
+  `migrations` (fns of the datasource, each creating and updating a group's own tables). Each
+  query opens its own connection (cheap for a local file); WAL mode plus a busy timeout let
+  concurrent requests wait on each other's writes instead of failing."
+  [path & migrations]
+  (when (str/blank? path)
+    (throw (ex-info "no database path configured (set DB_PATH)" {})))
+  (some-> (io/file path) .getAbsoluteFile .getParentFile .mkdirs)
+  (let [ds (jdbc/get-datasource {:dbtype "sqlite" :dbname path
+                                 :journal_mode "WAL" :busy_timeout 5000})]
+    (run! #(% ds) migrations)
+    ds))
+
 (defn new-id
   "An id for a row this app owns: a random UUID, as lowercase text. Never an integer: an LLM that
   reads `7` as a todo's id confuses it with a 7 in a title, and a UUID is nothing else."

src/tbd/todo/db.clj

@@ -1,22 +1,11 @@
 (ns tbd.todo.db
-  "SQLite persistence for the todo group, in a local file. Storage only: validation and other
+  "SQLite persistence for the todo group, in the app's database (see tbd.sqlite). Storage only: validation and other
   rules live in tbd.todo.core. Every query takes an `owner-id` and filters on it, so one
   user's todos can't be read or changed through another user's ids."
-  (:require [clojure.java.io :as io]
-            [clojure.string :as str]
-            [next.jdbc :as jdbc]
+  (:require [next.jdbc :as jdbc]
             [next.jdbc.result-set :as rs]
             [tbd.sqlite :as sqlite]))
 
-(def default-path "data/todo.db")
-
-(defn env-config
-  "TODO_DB_PATH: where the SQLite file lives (default data/todo.db; its directory is created)."
-  ([] (env-config (System/getenv)))
-  ([env]
-   (let [path (get env "TODO_DB_PATH")]
-     {:db-path (if (str/blank? path) default-path (str/trim path))})))
-
 (def ^:private create-todos
   ;; `id` is a UUID (see tbd.sqlite/new-id); `owner_id` is the user's id in Gitea, which the app
   ;; doesn't issue, so it stays the integer Gitea gave.
@@ -28,22 +17,18 @@
      created_at INTEGER NOT NULL,
      updated_at INTEGER NOT NULL)")
 
-(defn- migrate! [ds]
+(defn migrate!
+  "Creates the todo tables in the database `ds`, or brings them up to date."
+  [ds]
   (sqlite/migrate-integer-id! ds "todos" create-todos)
   (jdbc/execute! ds [create-todos])
   (jdbc/execute! ds ["CREATE INDEX IF NOT EXISTS todos_by_owner ON todos (owner_id, created_at)"]))
 
 (defn open
-  "A datasource for the database file at `path`, created and migrated if need be. Each query
-  opens its own connection (cheap for a local file); WAL mode plus a busy timeout let
-  concurrent requests wait on each other's writes instead of failing."
+  "A database with just the todo tables, at `path`; the app opens one with every group's tables
+  (see tbd.core), which this is for tests of the group on its own."
   [path]
-  (when (str/blank? path)
-    (throw (ex-info "no todo database path configured (set TODO_DB_PATH)" {})))
-  (some-> (io/file path) .getAbsoluteFile .getParentFile .mkdirs)
-  (doto (jdbc/get-datasource {:dbtype "sqlite" :dbname path
-                              :journal_mode "WAL" :busy_timeout 5000})
-    migrate!))
+  (sqlite/open path migrate!))
 
 (def ^:private opts {:builder-fn rs/as-unqualified-kebab-maps})
 

src/tbd/todo/routes.clj

@@ -39,7 +39,7 @@
     (reply 200 (fn [_] {:deleted true}) (service/delete! ds user id))))
 
 (defn routes
-  "`ds` is a delay of the todo database (see tbd.todo.db/open), shared with the MCP tools."
+  "`ds` is a delay of the app's database (see tbd.sqlite/open), shared with the MCP tools."
   [ds]
   (let [id-param {:path schema/IdParam}]
     ["/api/todos"

src/tbd/todo/service.clj

@@ -5,7 +5,7 @@
   same on both and are written (and tested) once. A front end only translates: its request into
   arguments, and the result into a response.
 
-  `ds` is the delay of the todo database the app shares (opened on first use, so a caller who is
+  `ds` is the delay of the database the app shares (opened on first use, so a caller who is
   turned away earlier never opens it). `user` is the authenticated caller, {:id ...} at least
   (however they proved who they are: a session or an access token); a todo is only ever reached
   through its owner's id, and someone else's is reported exactly like a missing one.