From 768b8e8d29eb198ad3369a95a1740ca9cf261845 Mon Sep 17 00:00:00 2001 From: Adam Jeniski Date: Tue, 3 Feb 2026 23:09:57 -0500 Subject: [PATCH] Add package-map support for cross-repo SCIP navigation - Add --package-map CLI option to specify namespace-to-package mappings - Update symbol format to include package coordinates for cross-repo nav - Skip external_symbols for packages that will be indexed separately - This enables Sourcegraph to resolve cross-repo references and show documentation from the external package's index Co-Authored-By: Claude Opus 4.5 --- .github/workflows/scip-index.yml | 2 +- package-map.edn | 4 + src/scip_clojure/core.clj | 202 +++++++++++++++++++++++++------ 3 files changed, 170 insertions(+), 38 deletions(-) create mode 100644 package-map.edn diff --git a/.github/workflows/scip-index.yml b/.github/workflows/scip-index.yml index ab21565..beb2538 100644 --- a/.github/workflows/scip-index.yml +++ b/.github/workflows/scip-index.yml @@ -32,7 +32,7 @@ jobs: run: clojure -T:build compile-java - name: Generate SCIP index - run: clojure -M:run -p . -o index.scip + run: clojure -M:run -p . -o index.scip -m package-map.edn env: CLOJURE_LSP_PATH: /usr/local/bin/clojure-lsp diff --git a/package-map.edn b/package-map.edn new file mode 100644 index 0000000..268e6c4 --- /dev/null +++ b/package-map.edn @@ -0,0 +1,4 @@ +;; Package map for cross-repository SCIP navigation +;; Maps namespace prefixes to package@version coordinates + +{"scip-clojure" "io.github.ajet/scip-clojure@main"} diff --git a/src/scip_clojure/core.clj b/src/scip_clojure/core.clj index 5db23a2..ce6123b 100644 --- a/src/scip_clojure/core.clj +++ b/src/scip_clojure/core.clj @@ -16,8 +16,60 @@ :default "."] ["-o" "--output FILE" "Output SCIP index file" :default "index.scip"] + ["-m" "--package-map FILE" "Package map JSON file for cross-repo navigation" + :default nil] ["-h" "--help" "Show help"]]) +;; --- Package map support for cross-repo navigation --- + +(defn load-package-map + "Load package map from EDN file. + Format: {\"ns-prefix\" \"package@version\", ...} + Example: {\"tui\" \"io.github.ajet/clojure-tui@main\"} + + The ns-prefix is matched against namespace names. A prefix of 'tui' + matches 'tui.core', 'tui.render', etc. + + Also supports JSON format for convenience." + [path] + (when path + (let [content (slurp path) + ;; Try to parse as EDN first, fall back to simple JSON conversion + parsed (try + (edn/read-string content) + (catch Exception _ + ;; Simple JSON to EDN conversion for basic object format + (edn/read-string + (-> content + (str/replace #"\":\s*\"" "\" \"") + (str/replace #",\s*\"" " \"") + (str/replace #",\s*\}" "}")))))] + ;; Ensure all keys and values are strings + (into {} + (map (fn [[k v]] [(str k) (str v)])) + parsed)))) + +(defn parse-package-spec + "Parse 'package@version' into [package version]. + If no @ present, uses empty string for version." + [spec] + (let [parts (str/split spec #"@" 2)] + [(first parts) (or (second parts) "")])) + +(defn find-package-for-ns + "Find the package spec for a namespace using the package map. + Returns [package version] or nil if not found. + Matches by longest prefix first." + [ns-sym package-map] + (when package-map + (let [ns-str (str ns-sym) + ;; Sort by prefix length descending for longest match + sorted-prefixes (->> (keys package-map) + (sort-by count) + reverse)] + (when-let [matching-prefix (first (filter #(str/starts-with? ns-str %) sorted-prefixes))] + (parse-package-spec (get package-map matching-prefix)))))) + (defn escape-identifier "Escape an identifier for SCIP symbol format. Identifiers containing characters other than [_+-$a-zA-Z0-9] must be @@ -28,28 +80,55 @@ s (str "`" (str/replace s "`" "``") "`")))) +;; Dynamic var to hold package map during indexing +(def ^:dynamic *package-map* nil) + (defn make-symbol "Create a SCIP symbol identifier from namespace and name. - Format: scip-clojure clojure . . - Uses namespace as package name for cross-repo navigation." + With package map: scip-clojure clojure /. + Without package map: scip-clojure clojure . . + + The package format enables cross-repo navigation by using Maven-style + coordinates that Sourcegraph can resolve to repositories." [ns-sym var-name] - (format "scip-clojure clojure %s . %s." - (escape-identifier ns-sym) - (escape-identifier var-name))) + (if-let [[pkg version] (find-package-for-ns ns-sym *package-map*)] + ;; With package map: use package coordinates + ;; Format: scheme manager package version descriptor + ;; Descriptor uses namespace/ prefix for namespacing within package + (format "scip-clojure clojure %s %s %s/%s." + (escape-identifier pkg) + (escape-identifier (or version ".")) + (escape-identifier ns-sym) + (escape-identifier var-name)) + ;; Without package map: fall back to namespace-only format + (format "scip-clojure clojure %s . %s." + (escape-identifier ns-sym) + (escape-identifier var-name)))) (defn make-ns-symbol "Create a SCIP symbol identifier for a namespace." [ns-sym] - (format "scip-clojure clojure %s . " - (escape-identifier ns-sym))) + (if-let [[pkg version] (find-package-for-ns ns-sym *package-map*)] + (format "scip-clojure clojure %s %s %s/" + (escape-identifier pkg) + (escape-identifier (or version ".")) + (escape-identifier ns-sym)) + (format "scip-clojure clojure %s . " + (escape-identifier ns-sym)))) (defn make-alias-symbol "Create a SCIP symbol identifier for a namespace alias. The alias is scoped to the namespace where it's defined." [from-ns alias-name] - (format "scip-clojure clojure %s . %s." - (escape-identifier from-ns) - (escape-identifier alias-name))) + (if-let [[pkg version] (find-package-for-ns from-ns *package-map*)] + (format "scip-clojure clojure %s %s %s/%s." + (escape-identifier pkg) + (escape-identifier (or version ".")) + (escape-identifier from-ns) + (escape-identifier alias-name)) + (format "scip-clojure clojure %s . %s." + (escape-identifier from-ns) + (escape-identifier alias-name)))) (defn position->range "Convert row/col positions to SCIP range format. @@ -161,27 +240,43 @@ (.setSymbol builder (make-ns-symbol name)) (.build builder))) -(defn ns-alias->occurrence - "Convert a namespace alias definition (e.g., :as ansi) to SCIP Occurrence. - Creates a Definition for the alias symbol, scoped to the defining namespace." - [{:keys [from alias] :as ns-alias}] - (let [builder (Scip$Occurrence/newBuilder)] - (.addAllRange builder (position->range ns-alias)) - (.setSymbol builder (make-alias-symbol from alias)) - (.setSymbolRoles builder (int Scip$SymbolRole/Definition_VALUE)) - (.build builder))) +(defn ns-alias-definition->occurrence + "Convert a namespace-usage with :as alias to SCIP Occurrence for alias definition. + Creates a Definition for the alias symbol, scoped to the defining namespace. + E.g., [tui.ansi :as ansi] creates definition at 'ansi' position. + Uses alias-row/alias-col positions from namespace-usages." + [{:keys [from alias alias-row alias-col alias-end-col] :as ns-usage}] + (when (and alias alias-row alias-col) + (let [builder (Scip$Occurrence/newBuilder) + start-line (int (dec alias-row)) + start-char (int (dec alias-col)) + end-char (int (dec (or alias-end-col (+ alias-col (count (str alias))))))] + (.addAllRange builder [(int start-line) (int start-char) (int end-char)]) + (.setSymbol builder (make-alias-symbol from alias)) + (.setSymbolRoles builder (int Scip$SymbolRole/Definition_VALUE)) + (.build builder)))) + +(defn ns-alias-definition->symbol-info + "Create SCIP SymbolInformation for a namespace alias." + [{:keys [from name alias]}] + (when alias + (let [builder (Scip$SymbolInformation/newBuilder)] + (.setSymbol builder (make-alias-symbol from alias)) + (.addDocumentation builder (str "```clojure\nAlias for " name "\n```")) + (.build builder)))) (defn alias-usage->occurrence "Convert a var-usage with an alias to SCIP Occurrence for the alias part. - E.g., for (ansi/style ...), creates an occurrence for 'ansi' referencing the alias." - [{:keys [from alias row col name-col]}] + E.g., for (ansi/style ...), creates an occurrence for 'ansi' referencing the alias. + The alias position is calculated from col (start of expression) using alias length." + [{:keys [from alias row col]}] (when alias (let [builder (Scip$Occurrence/newBuilder) - ;; alias spans from col to name-col - 2 (excluding the /) - alias-end-col (- name-col 1) + alias-str (str alias) start-line (int (dec row)) start-char (int (dec col)) - end-char (int (dec alias-end-col))] + ;; alias ends at col + length(alias), before the / + end-char (int (+ start-char (count alias-str)))] (.addAllRange builder [(int start-line) (int start-char) (int end-char)]) (.setSymbol builder (make-alias-symbol from alias)) (.build builder)))) @@ -252,14 +347,26 @@ (.addDocumentation builder doc)) (.build builder))) +(defn ns-in-package-map? + "Check if a namespace is covered by the package-map. + If so, it will be indexed separately and we shouldn't include it + in external_symbols (per SCIP spec: 'Leave this field empty if you + assume the external package will get indexed separately')." + [ns-sym] + (when *package-map* + (some #(str/starts-with? (str ns-sym) %) (keys *package-map*)))) + (defn collect-external-symbols "Collect all unique external symbol references from analysis. - Returns symbol info for all external symbols, with docs when available. - Essential for cross-repo navigation to dependencies." + Returns symbol info for external symbols that won't be indexed separately. + Skips symbols from packages in the package-map since those will have + their own SCIP indexes with full documentation." [analysis internal-namespaces] (->> (vals analysis) (mapcat :var-usages) (filter #(external-ns? (:to %) internal-namespaces)) + ;; Skip symbols from packages that will be indexed separately + (remove #(ns-in-package-map? (:to %))) (map (fn [{:keys [to name]}] [to name])) (filter (fn [[ns-sym _]] ns-sym)) ; Filter out nil namespaces (distinct) @@ -279,7 +386,7 @@ [uri file-analysis project-root] (let [builder (Scip$Document/newBuilder) relative-path (uri->relative-path uri project-root) - {:keys [var-definitions var-usages namespace-definitions namespace-usages namespace-alias]} file-analysis + {:keys [var-definitions var-usages namespace-definitions namespace-usages]} file-analysis ;; Build refer-map: for vars imported via :refer, map var-name -> target-namespace ;; These entries have :refer true and :to populated @@ -305,8 +412,10 @@ ;; Namespace usages ns-usage-occurrences (map ns-usage->occurrence (or namespace-usages [])) - ;; Namespace alias definitions (e.g., :as ansi) - ns-alias-occurrences (map ns-alias->occurrence (or namespace-alias [])) + ;; Namespace alias definitions from namespace-usages entries with :alias + ;; E.g., [tui.ansi :as ansi] creates definition for 'ansi' + ns-alias-def-occurrences (keep ns-alias-definition->occurrence (or namespace-usages [])) + ns-alias-symbols (keep ns-alias-definition->symbol-info (or namespace-usages [])) ;; Alias usages from var-usages (e.g., ansi in ansi/style) alias-usage-occs (keep alias-usage->occurrence (or var-usages []))] @@ -318,11 +427,11 @@ ;; Add all occurrences (doseq [occ (concat def-occurrences usage-occurrences ns-def-occurrences ns-usage-occurrences - ns-alias-occurrences alias-usage-occs)] + ns-alias-def-occurrences alias-usage-occs)] (.addOccurrences builder occ)) ;; Add symbol information - (doseq [sym (concat def-symbols ns-def-symbols)] + (doseq [sym (concat def-symbols ns-def-symbols ns-alias-symbols)] (.addSymbols builder sym)) (.build builder))) @@ -401,7 +510,16 @@ (println "Usage: scip-clojure [options]") (println) (println "Options:") - (println summary)) + (println summary) + (println) + (println "Package Map Format (JSON):") + (println " {\"ns-prefix\": \"package@version\", ...}") + (println) + (println "Example package-map.json for cross-repo navigation:") + (println " {") + (println " \"tui\": \"io.github.ajet/clojure-tui@main\",") + (println " \"lazygitclj\": \"io.github.ajet/lazygitclj@main\"") + (println " }")) errors (do @@ -413,13 +531,23 @@ (let [project-root (-> (:project-root options) io/file .getCanonicalPath) - output-file (:output options)] + output-file (:output options) + package-map-file (:package-map options) + package-map (when package-map-file + (println "Loading package map from" package-map-file) + (load-package-map package-map-file))] (println "Generating SCIP index for" project-root) + (when package-map + (println "Package mappings:") + (doseq [[prefix spec] package-map] + (println " " prefix "->" spec))) (try - (let [dump (run-clojure-lsp-dump project-root) - index (dump->scip-index dump)] - (write-scip-index index output-file) - (println "Done!")) + ;; Bind package map for symbol generation + (binding [*package-map* package-map] + (let [dump (run-clojure-lsp-dump project-root) + index (dump->scip-index dump)] + (write-scip-index index output-file) + (println "Done!"))) (catch Exception e (println "Error:" (.getMessage e)) (when-let [data (ex-data e)]