Initial commit: SCIP indexer for Clojure

Features: - Generate SCIP index from clojure-lsp dump - Namespace definitions and usages - Var definitions and usages - Namespace alias definitions and usage references - External symbol documentation for hover Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-03 16:49:10 -05:00
commit 3f669b422a
6 changed files with 23482 additions and 0 deletions
@@ -0,0 +1,388 @@
+(ns scip-clojure.core
+  (:require [clojure.edn :as edn]
+            [clojure.java.io :as io]
+            [clojure.java.shell :refer [sh]]
+            [clojure.string :as str]
+            [clojure.tools.cli :refer [parse-opts]]
+            [clojure.repl :as repl])
+  (:import [scip Scip$Index Scip$Document Scip$Occurrence Scip$SymbolInformation
+            Scip$Metadata Scip$Metadata$Builder Scip$ToolInfo Scip$SymbolRole
+            Scip$PositionEncoding Scip$ProtocolVersion]
+           [java.io FileOutputStream StringWriter])
+  (:gen-class))
+
+(def cli-options
+  [["-p" "--project-root PATH" "Path to the Clojure project root"
+    :default "."]
+   ["-o" "--output FILE" "Output SCIP index file"
+    :default "index.scip"]
+   ["-h" "--help" "Show help"]])
+
+(defn escape-identifier
+  "Escape an identifier for SCIP symbol format.
+   Identifiers containing characters other than [_+-$a-zA-Z0-9] must be
+   wrapped in backticks. Backticks within the identifier are doubled."
+  [s]
+  (let [s (str s)]
+    (if (re-matches #"[_+\-$a-zA-Z0-9]+" s)
+      s
+      (str "`" (str/replace s "`" "``") "`"))))
+
+(defn make-symbol
+  "Create a SCIP symbol identifier from namespace and name.
+   Format: scip-clojure clojure clojure . namespace/name.
+   No spaces around descriptor suffixes (/ and .)"
+  [ns-sym var-name]
+  (format "scip-clojure clojure clojure . %s/%s."
+          (escape-identifier ns-sym)
+          (escape-identifier var-name)))
+
+(defn make-ns-symbol
+  "Create a SCIP symbol identifier for a namespace."
+  [ns-sym]
+  (format "scip-clojure clojure clojure . %s/"
+          (escape-identifier ns-sym)))
+
+(defn make-alias-symbol
+  "Create a SCIP symbol identifier for a namespace alias.
+   The alias is scoped to the namespace where it's defined.
+   Format: scip-clojure clojure clojure . from-ns/alias."
+  [from-ns alias-name]
+  (format "scip-clojure clojure clojure . %s/%s."
+          (escape-identifier from-ns)
+          (escape-identifier alias-name)))
+
+(defn position->range
+  "Convert row/col positions to SCIP range format.
+   SCIP uses 0-indexed [startLine, startChar, endLine, endChar].
+   For single-line occurrences, uses compact [startLine, startChar, endChar].
+   Uses name-row/name-col when available for precise symbol positioning."
+  [{:keys [row col name-row name-col name-end-row name-end-col]}]
+  (let [;; Prefer name-row/name-col for precise symbol location
+        start-line (int (dec (or name-row row 1)))
+        start-char (int (dec (or name-col col 1)))
+        end-line (int (dec (or name-end-row name-row row 1)))
+        end-char (int (dec (or name-end-col (+ (or name-col col 1) 1))))]
+    (if (= start-line end-line)
+      ;; Compact format for single-line
+      [(int start-line) (int start-char) (int end-char)]
+      ;; Full format for multi-line
+      [(int start-line) (int start-char) (int end-line) (int end-char)])))
+
+(defn var-definition->occurrence
+  "Convert a clojure-lsp var-definition to SCIP Occurrence."
+  [{:keys [ns name doc] :as var-def}]
+  (let [builder (Scip$Occurrence/newBuilder)]
+    (.addAllRange builder (position->range var-def))
+    (.setSymbol builder (make-symbol ns name))
+    ;; Definition role = 1
+    (.setSymbolRoles builder (int Scip$SymbolRole/Definition_VALUE))
+    (.build builder)))
+
+(defn var-definition->symbol-info
+  "Convert a clojure-lsp var-definition to SCIP SymbolInformation."
+  [{:keys [ns name doc defined-by fixed-arities]}]
+  (let [builder (Scip$SymbolInformation/newBuilder)]
+    (.setSymbol builder (make-symbol ns name))
+    (when doc
+      (.addDocumentation builder doc))
+    (when defined-by
+      (.addDocumentation builder (str "Defined by: " defined-by)))
+    (when (seq fixed-arities)
+      (.addDocumentation builder (str "Arities: " (str/join ", " fixed-arities))))
+    (.build builder)))
+
+(defn var-usage->occurrence
+  "Convert a clojure-lsp var-usage to SCIP Occurrence (reference)."
+  [{:keys [to name] :as var-usage}]
+  (let [builder (Scip$Occurrence/newBuilder)]
+    (.addAllRange builder (position->range var-usage))
+    (.setSymbol builder (make-symbol to name))
+    ;; No role bits = reference
+    (.build builder)))
+
+(defn ns-definition->occurrence
+  "Convert a namespace definition to SCIP Occurrence."
+  [{:keys [name row col name-end-row name-end-col] :as ns-def}]
+  (let [builder (Scip$Occurrence/newBuilder)]
+    (.addAllRange builder (position->range ns-def))
+    (.setSymbol builder (make-ns-symbol name))
+    (.setSymbolRoles builder (int Scip$SymbolRole/Definition_VALUE))
+    (.build builder)))
+
+(defn ns-definition->symbol-info
+  "Convert a namespace definition to SCIP SymbolInformation."
+  [{:keys [name doc]}]
+  (let [builder (Scip$SymbolInformation/newBuilder)]
+    (.setSymbol builder (make-ns-symbol name))
+    (when doc
+      (.addDocumentation builder doc))
+    (.build builder)))
+
+(defn ns-usage->occurrence
+  "Convert a namespace usage (require/import) to SCIP Occurrence."
+  [{:keys [name] :as ns-usage}]
+  (let [builder (Scip$Occurrence/newBuilder)]
+    (.addAllRange builder (position->range ns-usage))
+    (.setSymbol builder (make-ns-symbol name))
+    (.build builder)))
+
+(defn ns-alias->occurrence
+  "Convert a namespace alias definition (e.g., :as ansi) to SCIP Occurrence.
+   Creates a Definition for the alias symbol, scoped to the defining namespace."
+  [{:keys [from alias] :as ns-alias}]
+  (let [builder (Scip$Occurrence/newBuilder)]
+    (.addAllRange builder (position->range ns-alias))
+    (.setSymbol builder (make-alias-symbol from alias))
+    (.setSymbolRoles builder (int Scip$SymbolRole/Definition_VALUE))
+    (.build builder)))
+
+(defn alias-usage->occurrence
+  "Convert a var-usage with an alias to SCIP Occurrence for the alias part.
+   E.g., for (ansi/style ...), creates an occurrence for 'ansi' referencing the alias."
+  [{:keys [from alias row col name-col]}]
+  (when alias
+    (let [builder (Scip$Occurrence/newBuilder)
+          ;; alias spans from col to name-col - 2 (excluding the /)
+          alias-end-col (- name-col 1)
+          start-line (int (dec row))
+          start-char (int (dec col))
+          end-char (int (dec alias-end-col))]
+      (.addAllRange builder [(int start-line) (int start-char) (int end-char)])
+      (.setSymbol builder (make-alias-symbol from alias))
+      (.build builder))))
+
+;; --- External symbol support for hover documentation ---
+
+(def ^:private external-ns-prefixes
+  "Namespace prefixes considered external (not project-internal)."
+  #{"clojure.core" "clojure.string" "clojure.set" "clojure.walk"
+    "clojure.zip" "clojure.data" "clojure.edn" "clojure.java.io"
+    "clojure.java.shell" "clojure.pprint" "clojure.test" "clojure.repl"
+    "clojure.spec.alpha" "clojure.spec.gen.alpha"})
+
+(defn- get-special-form-doc
+  "Get documentation for a special form from clojure.repl/special-doc-map.
+   This is the canonical source of truth for special form documentation."
+  [sym-name]
+  (when-let [doc-map (get @#'clojure.repl/special-doc-map (symbol sym-name))]
+    {:doc (:doc doc-map)
+     :arglists (:forms doc-map)  ;; special forms use :forms instead of :arglists
+     :special-form true}))
+
+(defn external-ns?
+  "Check if a namespace is external (standard library or dependency)."
+  [ns-sym internal-namespaces]
+  (let [ns-str (str ns-sym)]
+    (and (not (contains? internal-namespaces ns-sym))
+         (or (str/starts-with? ns-str "clojure.")
+             (str/starts-with? ns-str "java.")
+             (contains? external-ns-prefixes ns-str)))))
+
+(defn get-var-doc
+  "Get documentation for a var. Returns a map with :doc, :arglists, :macro.
+   Handles special forms which don't have var metadata by consulting
+   clojure.repl/special-doc-map (the canonical source of truth)."
+  [ns-sym var-name]
+  ;; First check if it's a special form (only in clojure.core)
+  (if-let [special-doc (and (= (str ns-sym) "clojure.core")
+                            (get-special-form-doc var-name))]
+    (assoc special-doc :name var-name :ns ns-sym)
+    ;; Otherwise try to resolve the var and get its metadata
+    (try
+      (require (symbol ns-sym))
+      (when-let [v (ns-resolve (symbol ns-sym) (symbol var-name))]
+        (let [m (meta v)]
+          {:doc (:doc m)
+           :arglists (:arglists m)
+           :macro (:macro m)
+           :name var-name
+           :ns ns-sym}))
+      (catch Exception _ nil))))
+
+(defn format-arglists
+  "Format arglists for display."
+  [arglists]
+  (when (seq arglists)
+    (str/join "\n" (map #(str "  " (pr-str %)) arglists))))
+
+(defn external-symbol->symbol-info
+  "Create SCIP SymbolInformation for an external symbol with documentation."
+  [{:keys [ns name doc arglists macro special-form]}]
+  (let [builder (Scip$SymbolInformation/newBuilder)
+        sig (str (cond special-form "special form "
+                       macro "macro "
+                       :else "")
+                 ns "/" name)]
+    (.setSymbol builder (make-symbol ns name))
+    ;; Add signature as first doc line
+    (.addDocumentation builder (str "```clojure\n" sig "\n```"))
+    ;; Add arglists
+    (when (seq arglists)
+      (.addDocumentation builder (str "```clojure\n" (format-arglists arglists) "\n```")))
+    ;; Add docstring
+    (when doc
+      (.addDocumentation builder doc))
+    (.build builder)))
+
+(defn collect-external-symbols
+  "Collect all unique external symbol references from analysis."
+  [analysis internal-namespaces]
+  (->> (vals analysis)
+       (mapcat :var-usages)
+       (filter #(external-ns? (:to %) internal-namespaces))
+       (map (fn [{:keys [to name]}] [to name]))
+       (distinct)
+       (keep (fn [[ns-sym var-name]]
+               (get-var-doc ns-sym var-name)))
+       (filter :doc)))  ; Only include symbols with documentation
+
+(defn uri->relative-path
+  "Convert file:// URI to relative path from project root."
+  [uri project-root]
+  (let [path (str/replace-first uri "file://" "")]
+    (str/replace-first path (str project-root "/") "")))
+
+(defn file-analysis->document
+  "Convert a single file's analysis to a SCIP Document."
+  [uri file-analysis project-root]
+  (let [builder (Scip$Document/newBuilder)
+        relative-path (uri->relative-path uri project-root)
+        {:keys [var-definitions var-usages namespace-definitions namespace-usages namespace-alias]} file-analysis
+
+        ;; Create occurrences for definitions
+        def-occurrences (map var-definition->occurrence (or var-definitions []))
+
+        ;; Create symbol info for definitions
+        def-symbols (map var-definition->symbol-info (or var-definitions []))
+
+        ;; Create occurrences for usages (references)
+        usage-occurrences (map var-usage->occurrence (or var-usages []))
+
+        ;; Namespace definitions
+        ns-def-occurrences (map ns-definition->occurrence (or namespace-definitions []))
+        ns-def-symbols (map ns-definition->symbol-info (or namespace-definitions []))
+
+        ;; Namespace usages
+        ns-usage-occurrences (map ns-usage->occurrence (or namespace-usages []))
+
+        ;; Namespace alias definitions (e.g., :as ansi)
+        ns-alias-occurrences (map ns-alias->occurrence (or namespace-alias []))
+
+        ;; Alias usages from var-usages (e.g., ansi in ansi/style)
+        alias-usage-occs (keep alias-usage->occurrence (or var-usages []))]
+
+    (.setLanguage builder "clojure")
+    (.setRelativePath builder relative-path)
+    (.setPositionEncoding builder Scip$PositionEncoding/UTF8CodeUnitOffsetFromLineStart)
+
+    ;; Add all occurrences
+    (doseq [occ (concat def-occurrences usage-occurrences
+                        ns-def-occurrences ns-usage-occurrences
+                        ns-alias-occurrences alias-usage-occs)]
+      (.addOccurrences builder occ))
+
+    ;; Add symbol information
+    (doseq [sym (concat def-symbols ns-def-symbols)]
+      (.addSymbols builder sym))
+
+    (.build builder)))
+
+(defn dump->scip-index
+  "Convert clojure-lsp dump data to a SCIP Index."
+  [{:keys [analysis project-root] :as dump}]
+  (let [builder (Scip$Index/newBuilder)
+        metadata-builder (Scip$Metadata/newBuilder)
+        tool-info-builder (Scip$ToolInfo/newBuilder)
+        ;; Collect internal namespaces from namespace-definitions
+        internal-namespaces (->> (vals analysis)
+                                 (mapcat :namespace-definitions)
+                                 (map :name)
+                                 (set))]
+
+    ;; Set tool info
+    (.setName tool-info-builder "scip-clojure")
+    (.setVersion tool-info-builder "0.2.0")
+
+    ;; Set metadata
+    (.setVersion metadata-builder Scip$ProtocolVersion/UnspecifiedProtocolVersion)
+    (.setToolInfo metadata-builder (.build tool-info-builder))
+    (.setProjectRoot metadata-builder (str "file://" project-root))
+
+    (.setMetadata builder (.build metadata-builder))
+
+    ;; Convert each file
+    (doseq [[uri file-analysis] analysis]
+      (let [doc (file-analysis->document (str uri) file-analysis project-root)]
+        (.addDocuments builder doc)))
+
+    ;; Add external symbols with documentation for hover
+    (println "Collecting external symbol documentation...")
+    (let [external-syms (collect-external-symbols analysis internal-namespaces)]
+      (println "Found" (count external-syms) "external symbols with documentation")
+      (doseq [sym external-syms]
+        (.addExternalSymbols builder (external-symbol->symbol-info sym))))
+
+    (.build builder)))
+
+(defn run-clojure-lsp-dump
+  "Run clojure-lsp dump command and return parsed EDN."
+  [project-root]
+  (println "Running clojure-lsp dump...")
+  (let [lsp-path (or (System/getenv "CLOJURE_LSP_PATH")
+                     (str (System/getProperty "user.home") "/.local/bin/clojure-lsp")
+                     "clojure-lsp")
+        result (sh lsp-path "dump"
+                   "--project-root" project-root
+                   "--output" "{:format :edn}")]
+    (if (zero? (:exit result))
+      (do
+        (println "Parsing clojure-lsp output...")
+        (edn/read-string (:out result)))
+      (throw (ex-info "clojure-lsp dump failed"
+                      {:exit (:exit result)
+                       :err (:err result)})))))
+
+(defn write-scip-index
+  "Write SCIP Index to a file."
+  [index output-path]
+  (with-open [out (FileOutputStream. output-path)]
+    (.writeTo index out))
+  (println "Wrote SCIP index to" output-path))
+
+(defn -main
+  [& args]
+  (let [{:keys [options errors summary]} (parse-opts args cli-options)]
+    (cond
+      (:help options)
+      (do
+        (println "scip-clojure - Generate SCIP index from Clojure project")
+        (println)
+        (println "Usage: scip-clojure [options]")
+        (println)
+        (println "Options:")
+        (println summary))
+
+      errors
+      (do
+        (doseq [err errors]
+          (println "Error:" err))
+        (System/exit 1))
+
+      :else
+      (let [project-root (-> (:project-root options)
+                             io/file
+                             .getCanonicalPath)
+            output-file (:output options)]
+        (println "Generating SCIP index for" project-root)
+        (try
+          (let [dump (run-clojure-lsp-dump project-root)
+                index (dump->scip-index dump)]
+            (write-scip-index index output-file)
+            (println "Done!"))
+          (catch Exception e
+            (println "Error:" (.getMessage e))
+            (when-let [data (ex-data e)]
+              (when (:err data)
+                (println "stderr:" (:err data))))
+            (System/exit 1)))))))