init research
This commit is contained in:
+2509
File diff suppressed because it is too large
Load Diff
+105
@@ -0,0 +1,105 @@
|
||||
(ns tech.v3.libs.arrow.liblz4
|
||||
(:require [tech.v3.datatype :as dt]
|
||||
[tech.v3.datatype.ffi :as dt-ffi]
|
||||
[tech.v3.datatype.ffi.size-t :as ffi-size-t]
|
||||
[tech.v3.datatype.struct :as dt-struct]
|
||||
[tech.v3.resource :as resource])
|
||||
(:import [tech.v3.datatype.ffi Pointer]))
|
||||
|
||||
|
||||
(def ^{:tag 'long} LZ4F_default 0)
|
||||
(def ^{:tag 'long} LZ4F_max64KB 4)
|
||||
(def ^{:tag 'long} LZ4F_max256KB 5)
|
||||
(def ^{:tag 'long} LZ4F_max1MB 6)
|
||||
(def ^{:tag 'long} LZ4F_max4MB 7)
|
||||
(def ^{:tag 'long} LZ4F_VERSION 100)
|
||||
(def ^{:tag 'long} LZ4F_HEADER_SIZE_MAX 19)
|
||||
(def ^{:tag 'long} LZ4F_blockLinked 0)
|
||||
(def ^{:tag 'long} LZ4F_blockIndependent 1)
|
||||
(def ^{:tag 'long} LZ4F_noContentChecksum 0)
|
||||
(def ^{:tag 'long} LZ4F_contentChecksumEnabled 1)
|
||||
(def ^{:tag 'long} LZ4F_noBlockChecksum 0)
|
||||
(def ^{:tag 'long} LZ4F_blockChecksumEnabled 1)
|
||||
(def ^{:tag 'long} LZ4F_frame 0)
|
||||
(def ^{:tag 'long} LZ4F_skippableFrame 1)
|
||||
|
||||
|
||||
(dt-ffi/define-library!
|
||||
lz4
|
||||
'{:LZ4F_getVersion {:rettype :int32
|
||||
:doc "library version"}
|
||||
:LZ4F_createDecompressionContext {:rettype :size-t
|
||||
:argtypes [[dctxPtr :pointer] ;;ptrptr
|
||||
[version :int32]]}
|
||||
:LZ4F_freeDecompressionContext {:rettype :size-t
|
||||
:argtypes [[dctx :pointer]]}
|
||||
|
||||
:LZ4F_isError {:rettype :int32
|
||||
:argtypes [[err :size-t]]}
|
||||
:LZ4F_getErrorName {:rettype :string
|
||||
:argtypes [[err :size-t]]}
|
||||
:LZ4F_getFrameInfo {:rettype :size-t
|
||||
:argtypes [[dctx :pointer] ;;decompression context
|
||||
[frameInfoPtr :pointer] ;;frameinfo*
|
||||
[srcBuffer :pointer] ;;void*
|
||||
[srcSizePtr :pointer]]} ;;size-t*
|
||||
:LZ4F_decompress {:rettype :size-t
|
||||
:argtypes [[dctx :pointer]
|
||||
[dstbuf :pointer]
|
||||
[dstsize :pointer] ;;size-t*
|
||||
[srcbuf :pointer]
|
||||
[srcsize :pointer] ;;size-t*
|
||||
[decomp-opt :pointer?]]} ;;dOptPtr
|
||||
|
||||
}
|
||||
nil
|
||||
nil)
|
||||
|
||||
;;checked against actual record layout
|
||||
(def frame-info-def*
|
||||
(delay (dt-struct/define-datatype! :lz4-frame-info
|
||||
[{:name :block-size-id :datatype :int32}
|
||||
{:name :block-mode :datatype :int32}
|
||||
{:name :content-checksum :datatype :int32}
|
||||
{:name :frame-type :datatype :int32}
|
||||
{:name :content-size :datatype :int64}
|
||||
{:name :dict-id :datatype :int32}
|
||||
{:name :checksum :datatype :int32}])))
|
||||
|
||||
|
||||
(def decompress-options-def*
|
||||
(delay (dt-struct/define-datatype! :lz4-decomp-options
|
||||
[{:name :stable-dst :datatype :int32}
|
||||
{:name :reserved0 :datatype :int32}
|
||||
{:name :reserved1 :datatype :int32}
|
||||
{:name :reserved2 :datatype :int32}])))
|
||||
|
||||
|
||||
;; Used for graal native pathways
|
||||
(defn set-library-instance!
|
||||
[lib-instance]
|
||||
(dt-ffi/library-singleton-set-instance! lz4 lib-instance))
|
||||
|
||||
|
||||
(defn initialize!
|
||||
[]
|
||||
(dt-ffi/library-singleton-set! lz4 "lz4"))
|
||||
|
||||
|
||||
(defn create-decomp-ctx
|
||||
[]
|
||||
(let [retval
|
||||
(resource/stack-resource-context
|
||||
(let [ptrptr (dt-ffi/make-ptr :pointer 0)
|
||||
ctx-err (LZ4F_createDecompressionContext ptrptr LZ4F_VERSION)]
|
||||
(when (== 1 (long (LZ4F_isError ctx-err)))
|
||||
(throw (Exception. (str (LZ4F_getErrorName ctx-err)))))
|
||||
(Pointer. (ptrptr 0))))
|
||||
addr (.address retval)]
|
||||
(resource/track retval {:track-type :auto
|
||||
:dispose-fn #(LZ4F_freeDecompressionContext (Pointer. addr))})))
|
||||
|
||||
|
||||
(defn free-decomp-ctx
|
||||
[ctx]
|
||||
(LZ4F_freeDecompressionContext ctx))
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
(ns tech.v3.libs.clj-transit
|
||||
"Transit bindings for the jvm version of tech.v3.dataset."
|
||||
(:require [tech.v3.dataset :as ds]
|
||||
[tech.v3.dataset.impl.dataset :as ds-impl]
|
||||
[tech.v3.dataset.io :as ds-io]
|
||||
[tech.v3.dataset.dynamic-int-list :as int-list]
|
||||
[tech.v3.io :as io]
|
||||
[tech.v3.datatype :as dtype]
|
||||
[tech.v3.datatype.errors :as errors]
|
||||
[tech.v3.datatype.casting :as casting]
|
||||
[tech.v3.datatype.packing :as packing]
|
||||
[tech.v3.datatype.datetime :as dtype-dt]
|
||||
[tech.v3.datatype.array-buffer :as abuf]
|
||||
[tech.v3.datatype.bitmap :as bitmap]
|
||||
[clj-commons.primitive-math :as pmath]
|
||||
[ham-fisted.api :as hamf]
|
||||
[cognitect.transit :as t])
|
||||
(:import [tech.v3.dataset.impl.dataset Dataset]
|
||||
[tech.v3.dataset.impl.column Column]
|
||||
[tech.v3.dataset Text]
|
||||
[java.nio ByteBuffer ByteOrder]
|
||||
[java.nio.charset StandardCharsets]
|
||||
[java.util Base64 HashMap]
|
||||
[java.time LocalDate Instant]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
|
||||
(defn- le-bbuf
|
||||
^ByteBuffer [n-elems]
|
||||
(-> (ByteBuffer/wrap (byte-array n-elems))
|
||||
(.order ByteOrder/LITTLE_ENDIAN)))
|
||||
|
||||
(defn- numeric-data->b64
|
||||
[col]
|
||||
(let [src-data (dtype/->reader col)
|
||||
n-elems (count col)
|
||||
^ByteBuffer bbuf
|
||||
(case (casting/host-flatten (dtype/elemwise-datatype col))
|
||||
:int8
|
||||
(let [retval (le-bbuf n-elems)
|
||||
dst-data (dtype/->buffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.writeByte dst-data idx (unchecked-byte (.readLong src-data idx))))
|
||||
retval)
|
||||
:int16
|
||||
(let [retval (le-bbuf (* Short/BYTES n-elems))
|
||||
dst-data (.asShortBuffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.put dst-data idx (unchecked-short (.readLong src-data idx))))
|
||||
retval)
|
||||
:int32
|
||||
(let [retval (le-bbuf (* Integer/BYTES n-elems))
|
||||
dst-data (.asIntBuffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.put dst-data idx (unchecked-int (.readLong src-data idx))))
|
||||
retval)
|
||||
:int64
|
||||
(let [retval (le-bbuf (* Long/BYTES n-elems))
|
||||
dst-data (.asLongBuffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.put dst-data idx (.readLong src-data idx)))
|
||||
retval)
|
||||
:float32
|
||||
(let [retval (le-bbuf (* Float/BYTES n-elems))
|
||||
dst-data (.asFloatBuffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.put dst-data idx (.readDouble src-data idx)))
|
||||
retval)
|
||||
:float64
|
||||
(let [retval (le-bbuf (* Double/BYTES n-elems))
|
||||
dst-data (.asDoubleBuffer retval)]
|
||||
(dotimes [idx (count col)]
|
||||
(.put dst-data idx (.readDouble src-data idx)))
|
||||
retval))]
|
||||
(String. (.encode (Base64/getEncoder) (.array bbuf)))))
|
||||
|
||||
|
||||
(defn- string-col->data
|
||||
[col]
|
||||
(let [strdata (dtype/make-list :string)
|
||||
indexes (dtype/make-list :int32)
|
||||
seen (HashMap.)]
|
||||
(dotimes [idx (count col)]
|
||||
(let [strval (str (or (col idx) ""))
|
||||
idx (.computeIfAbsent seen strval (reify java.util.function.Function
|
||||
(apply [this arg]
|
||||
(let [retval (count strdata)]
|
||||
(.add strdata strval)
|
||||
retval))))]
|
||||
(.add indexes idx)))
|
||||
{:strtable (hamf/object-array strdata)
|
||||
:indexes (numeric-data->b64 indexes)}))
|
||||
|
||||
|
||||
;;The assumption here is that transit has already optimized a vector or strings.
|
||||
(defn- text-col->data
|
||||
[col]
|
||||
(let [offsets (int-list/dynamic-int-list 0)
|
||||
data (StringBuilder.)
|
||||
n-elems (dtype/ecount col)]
|
||||
(dotimes [idx n-elems]
|
||||
(let [next-data (str (or (col idx) ""))]
|
||||
(.append data next-data)
|
||||
(.addLong offsets (.length data))))
|
||||
(let [offbuf (dtype/as-concrete-buffer offsets)]
|
||||
{:offset-dtype (dtype/elemwise-datatype offbuf)
|
||||
:offsets (numeric-data->b64 offbuf)
|
||||
:buffer (.toString data)})))
|
||||
|
||||
|
||||
(defn- obj-col->numeric-b64
|
||||
[col dst-dtype convert-fn]
|
||||
(let [data (dtype/make-list dst-dtype)]
|
||||
(dotimes [idx (count col)]
|
||||
(.add data (if-let [colval (col idx)]
|
||||
(convert-fn colval)
|
||||
0)))
|
||||
(numeric-data->b64 data)))
|
||||
|
||||
|
||||
(defn- col->data
|
||||
[col]
|
||||
(let [col-dt (packing/unpack-datatype (dtype/elemwise-datatype col))]
|
||||
{:metadata (assoc (meta col) :datatype col-dt)
|
||||
:missing (vec (-> (ds/missing col)
|
||||
(bitmap/->random-access)
|
||||
(hamf/object-array)
|
||||
vec))
|
||||
:data
|
||||
(cond
|
||||
(casting/numeric-type? col-dt)
|
||||
(numeric-data->b64 col)
|
||||
(= :boolean col-dt)
|
||||
(numeric-data->b64 (dtype/make-reader :uint8 (count col)
|
||||
(if (col idx) 1 0)))
|
||||
(= :string col-dt)
|
||||
(string-col->data col)
|
||||
(= :text col-dt)
|
||||
(text-col->data col)
|
||||
(#{:packed-local-date :local-date} col-dt)
|
||||
(obj-col->numeric-b64 col :int32 dtype-dt/local-date->days-since-epoch)
|
||||
(#{:packed-instant :instant :packed-milli-instant} col-dt)
|
||||
(obj-col->numeric-b64 col :int64 dtype-dt/instant->milliseconds-since-epoch)
|
||||
:else ;;Punt!!
|
||||
(vec col))}))
|
||||
|
||||
|
||||
(defn ^:no-doc dataset->data
|
||||
"Dataset to transit-safe data"
|
||||
[ds]
|
||||
{:metadata (meta ds)
|
||||
:flavor :transit
|
||||
:version 1
|
||||
:columns (mapv col->data (ds/columns ds))})
|
||||
|
||||
|
||||
(defn ^:no-doc b64->numeric-data
|
||||
[^String data dtype]
|
||||
(let [byte-data (.decode (Base64/getDecoder) (.getBytes data))
|
||||
bbuf (-> (ByteBuffer/wrap byte-data)
|
||||
(.order ByteOrder/LITTLE_ENDIAN))]
|
||||
(case dtype
|
||||
:int8 (dtype/->buffer byte-data)
|
||||
:uint8 (dtype/make-reader :uint8 (alength byte-data)
|
||||
(pmath/byte->ubyte (aget byte-data idx)))
|
||||
:int16 (let [sbuf (.asShortBuffer bbuf)]
|
||||
(dtype/make-reader :int16 (.limit sbuf) (.get sbuf idx)))
|
||||
:uint16 (let [sbuf (.asShortBuffer bbuf)]
|
||||
(dtype/make-reader :uint16 (.limit sbuf) (pmath/short->ushort
|
||||
(.get sbuf idx))))
|
||||
:int32 (let [ibuf (.asIntBuffer bbuf)]
|
||||
(dtype/make-reader :int32 (.limit ibuf) (.get ibuf idx)))
|
||||
:uint32 (let [ibuf (.asIntBuffer bbuf)]
|
||||
(dtype/make-reader :uint32 (.limit ibuf) (pmath/int->uint (.get ibuf idx))))
|
||||
:int64 (let [lbuf (.asLongBuffer bbuf)]
|
||||
(dtype/make-reader :int64 (.limit lbuf) (.get lbuf idx)))
|
||||
:uint64 (let [lbuf (.asLongBuffer bbuf)]
|
||||
(dtype/make-reader :uint64 (.limit lbuf) (.get lbuf idx)))
|
||||
:float32 (let [fbuf (.asFloatBuffer bbuf)]
|
||||
(dtype/make-reader :float32 (.limit fbuf) (.get fbuf idx)))
|
||||
:float64 (let [fbuf (.asDoubleBuffer bbuf)]
|
||||
(dtype/make-reader :float64 (.limit fbuf) (.get fbuf idx))))))
|
||||
|
||||
|
||||
(defn ^:no-doc str-data->coldata
|
||||
[{:keys [strtable indexes]}]
|
||||
(let [indexes (dtype/->reader (b64->numeric-data indexes :int32))
|
||||
strdata (dtype/->reader strtable)]
|
||||
(dtype/make-reader :string (count indexes) (strdata (indexes idx)))))
|
||||
|
||||
|
||||
(defn ^:no-doc text-data->coldata
|
||||
[{:keys [offsets buffer offset-dtype]}]
|
||||
(let [offsets (dtype/->buffer (b64->numeric-data offsets offset-dtype))
|
||||
n-elems (dtype/ecount offsets)
|
||||
buffer (str buffer)]
|
||||
(-> (dtype/make-reader
|
||||
:text n-elems
|
||||
(let [prev-idx (unchecked-dec idx)
|
||||
prev-offset (long (if (> prev-idx -1)
|
||||
(.readLong offsets prev-idx)
|
||||
0))]
|
||||
(Text. (.substring buffer prev-offset (.readLong offsets idx))))))))
|
||||
|
||||
|
||||
(defn ^:no-doc data->dataset
|
||||
[{:keys [columns] :as ds-data}]
|
||||
(let [ds-meta (:metadata ds-data)]
|
||||
(errors/when-not-errorf
|
||||
(and ds-meta columns)
|
||||
"Passed in data does not appear to have metadata or columns")
|
||||
(->> (:columns ds-data)
|
||||
(map
|
||||
(fn [{:keys [metadata missing data]}]
|
||||
(let [dtype (:datatype metadata)]
|
||||
#:tech.v3.dataset{:metadata metadata
|
||||
:missing (bitmap/->bitmap missing)
|
||||
;;do not re-scan data.
|
||||
:force-datatype? true
|
||||
:data
|
||||
(cond
|
||||
(casting/numeric-type? dtype)
|
||||
(b64->numeric-data data dtype)
|
||||
(= :boolean dtype)
|
||||
(let [ibuf (b64->numeric-data data :int8)]
|
||||
(dtype/make-reader :boolean (count ibuf)
|
||||
(if (== 0 (unchecked-long (ibuf idx)))
|
||||
false true)))
|
||||
(= :string dtype)
|
||||
(str-data->coldata data)
|
||||
(= :text dtype)
|
||||
(text-data->coldata data)
|
||||
(= :local-date dtype)
|
||||
(-> (b64->numeric-data data :int32)
|
||||
(dtype/->array-buffer)
|
||||
(abuf/set-datatype :packed-local-date))
|
||||
(= :instant dtype)
|
||||
(-> (b64->numeric-data data :int64)
|
||||
(dtype/->array-buffer)
|
||||
(abuf/set-datatype :packed-milli-instant))
|
||||
:else
|
||||
(dtype/make-container dtype data))
|
||||
:name (:name metadata)})))
|
||||
(ds-impl/new-dataset {:dataset-name (:name ds-meta)} ds-meta))))
|
||||
|
||||
(comment
|
||||
;; verify that metadata is preserved
|
||||
(let [ds_a (ds/->dataset {:a [5]} {:parser-fn {:a :uint8} :dataset-name "keep-this-name"})
|
||||
ds_b (-> ds_a dataset->data data->dataset)]
|
||||
(and (= ds_a ds_b)
|
||||
(= (meta ds_a) (meta ds_b))))
|
||||
)
|
||||
|
||||
(def ^{:doc "Transit handler for writing datasets."}
|
||||
write-handlers {Dataset (t/write-handler "tech.v3.dataset" dataset->data)})
|
||||
(def ^{:doc "Transit handler for reading datasets."}
|
||||
read-handlers {"tech.v3.dataset" (t/read-handler data->dataset)})
|
||||
|
||||
(def ^{:doc "Transit write handlers for java.time.LocalDate and java.time.Instant"}
|
||||
java-time-write-handlers
|
||||
{LocalDate (t/write-handler "java.time.LocalDate" dtype-dt/local-date->days-since-epoch)
|
||||
Instant (t/write-handler "java.time.Instant" dtype-dt/instant->milliseconds-since-epoch)})
|
||||
|
||||
(def ^{:doc "Transit read handlers for java.time.LocalDate and java.time.Instant"}
|
||||
java-time-read-handlers
|
||||
{"java.time.LocalDate" (t/read-handler dtype-dt/days-since-epoch->local-date)
|
||||
"java.time.Instant" (t/read-handler dtype-dt/milliseconds-since-epoch->instant)})
|
||||
|
||||
|
||||
(defn dataset->transit
|
||||
"Convert a dataset into a transit encoded writer. See source for details."
|
||||
[ds out & [format handlers]]
|
||||
(let [writer (t/writer out (or format :json) {:handlers (merge write-handlers handlers)})]
|
||||
(t/write writer ds)))
|
||||
|
||||
|
||||
(defn dataset->transit-str
|
||||
"Convert a dataset to a transit-encoded json string. See [[dataset->transit]]."
|
||||
[ds & [format handlers]]
|
||||
(let [out (java.io.ByteArrayOutputStream.)]
|
||||
(dataset->transit ds out format handlers)
|
||||
(String. (.toByteArray out))))
|
||||
|
||||
|
||||
(defn transit->dataset
|
||||
"Decode a transite reader installing the transit handlers for datasets. See source
|
||||
for details."
|
||||
[in & [format handlers]]
|
||||
(let [reader (t/reader in (or format :json) {:handlers (merge read-handlers handlers)})]
|
||||
(t/read reader)))
|
||||
|
||||
|
||||
(defn transit-str->dataset
|
||||
"Convert a transit string to a dataset. This simply decodes the string, it isn't
|
||||
dataset specific. See [[transit->dataset]]"
|
||||
[ds-str]
|
||||
(transit->dataset (java.io.ByteArrayInputStream. (.getBytes ^String ds-str))))
|
||||
|
||||
|
||||
(defmethod ds-io/data->dataset :transit-json
|
||||
[data options]
|
||||
(with-open [in (io/input-stream data)]
|
||||
(transit->dataset in (:transit-format options)
|
||||
(:transit-read-handlers options))))
|
||||
|
||||
|
||||
(defmethod ds-io/dataset->data! :transit-json
|
||||
[ds output options]
|
||||
(with-open [outs (io/output-stream! output)]
|
||||
(dataset->transit ds outs
|
||||
(:transit-format options)
|
||||
(:transit-write-handlers options))))
|
||||
|
||||
(comment
|
||||
(defn master-ds
|
||||
[]
|
||||
(ds/->dataset (array-map :a (mapv double (range 5))
|
||||
:b (repeat 5 :a)
|
||||
:c (repeat 5 "hey")
|
||||
:d (repeat 5 {:a 1 :b 2})
|
||||
:e (repeat 4 [1 2 3])
|
||||
:f (repeat 5 (dtype-dt/local-date))
|
||||
:g (repeat 5 (dtype-dt/instant))
|
||||
:h [true false true true false]
|
||||
:i (repeat 5 "text")
|
||||
:j [1 nil 2 nil 3])
|
||||
{:parser-fn {:i :text}}))
|
||||
|
||||
|
||||
|
||||
(-> (master-ds)
|
||||
(dataset->transit-str)
|
||||
(transit-str->dataset))
|
||||
|
||||
|
||||
(def test-data (vec (repeatedly 100000 #(hash-map :time (rand)
|
||||
:temp (rand)
|
||||
:valid? (if (> (rand) 0.5)
|
||||
true
|
||||
false)))))
|
||||
|
||||
(def test-ds (ds/->dataset test-data))
|
||||
|
||||
(do
|
||||
|
||||
(defn t->file
|
||||
[ds fname]
|
||||
(with-open [outs (io/output-stream! fname)]
|
||||
(dataset->transit ds outs)))
|
||||
|
||||
(defn file->t
|
||||
[fname]
|
||||
(with-open [ins (io/input-stream fname)]
|
||||
(transit->dataset ins)))
|
||||
)
|
||||
|
||||
(time (t->file test-data "mapseq.transit-json"))
|
||||
;; 1027ms, 6.3MB raw, 1.9MB gzipped
|
||||
|
||||
(time (t->file test-ds "ds.transit-json"))
|
||||
;; 16ms, 2.2MB raw, 1.6MB gzipped
|
||||
|
||||
(do
|
||||
(def stocks (ds/->dataset "https://github.com/techascent/tech.ml.dataset/raw/master/test/data/stocks.csv" {:key-fn keyword}))
|
||||
|
||||
(io/make-parents "test/data/stocks.transit-json")
|
||||
(t->file stocks "test/data/stocks.transit-json")
|
||||
)
|
||||
|
||||
(require '[tech.v3.datatype.functional :as dfn])
|
||||
|
||||
(->> (ds/group-by-column stocks :symbol)
|
||||
(map (fn [[k v]]
|
||||
[k (-> (dfn/mean (v :price))
|
||||
(* 100)
|
||||
(Math/round)
|
||||
(/ 100.0))]))
|
||||
(into {}))
|
||||
;;{"MSFT" 24.74, "AMZN" 47.99, "IBM" 91.26, "GOOG" 415.87, "AAPL" 64.73}
|
||||
|
||||
|
||||
)
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
(ns tech.v3.libs.fastexcel
|
||||
"Parse a dataset in xlsx format. This namespace auto-registers a handler for
|
||||
the 'xlsx' file type so that when using ->dataset, `xlsx` will automatically map to
|
||||
`(first (workbook->datasets))`.
|
||||
|
||||
Note that this namespace does **not** auto-register a handler for the `xls` file type.
|
||||
`xls` is handled by the poi namespace.
|
||||
|
||||
|
||||
If you have an xlsx file that contains multiple sheets and you want a dataset
|
||||
out of each sheet you have to use `workbook->datasets` as opposed to the higher level
|
||||
`->dataset` operator.
|
||||
|
||||
This dependency has a known conflict with cognitec's aws library - see [issue 283](https://github.com/techascent/tech.ml.dataset/issues/283).
|
||||
|
||||
Required Dependencies:
|
||||
|
||||
```clojure
|
||||
[org.dhatim/fastexcel-reader \"0.12.8\" :exclusions [org.apache.poi/poi-ooxml]]
|
||||
```"
|
||||
(:require [tech.v3.io :as io]
|
||||
[tech.v3.datatype.protocols :as dtype-proto]
|
||||
[tech.v3.datatype :as dtype]
|
||||
[tech.v3.datatype.errors :as errors]
|
||||
[tech.v3.dataset.io.spreadsheet :as parse-spreadsheet]
|
||||
[tech.v3.dataset.io :as ds-io])
|
||||
(:import [org.dhatim.fastexcel.reader ReadableWorkbook
|
||||
Sheet Row Cell CellType]
|
||||
[tech.v3.dataset Spreadsheet$Workbook Spreadsheet$Sheet
|
||||
Spreadsheet$Row Spreadsheet$Cell]))
|
||||
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
|
||||
(defn- cell-type->keyword
|
||||
[^CellType cell-type]
|
||||
(condp = cell-type
|
||||
CellType/EMPTY :none
|
||||
CellType/NUMBER :float64
|
||||
CellType/BOOLEAN :boolean
|
||||
CellType/STRING :string))
|
||||
|
||||
|
||||
(defn- formula-type?
|
||||
[cell-type]
|
||||
(or (= cell-type CellType/FORMULA)
|
||||
(= cell-type CellType/ERROR)))
|
||||
|
||||
|
||||
(defn- try-parse-double
|
||||
[^Cell cell]
|
||||
(try
|
||||
(Double/parseDouble (.getRawValue cell))
|
||||
(catch Exception _e
|
||||
(.getRawValue cell))))
|
||||
|
||||
|
||||
(defn- wrap-cell
|
||||
[^Cell cell]
|
||||
(reify
|
||||
dtype-proto/PElemwiseDatatype
|
||||
(elemwise-datatype [this]
|
||||
(let [cell-type (.getType cell)]
|
||||
(if (formula-type? cell-type)
|
||||
(if (number? (try-parse-double cell))
|
||||
:float64
|
||||
:string)
|
||||
(cell-type->keyword cell-type))))
|
||||
Spreadsheet$Cell
|
||||
(getColumnNum [this] (.getColumnIndex cell))
|
||||
(missing [this] (= :none (dtype/get-datatype this)))
|
||||
(value [this]
|
||||
(if (formula-type? (.getType cell))
|
||||
(try-parse-double cell)
|
||||
(case (dtype-proto/elemwise-datatype this)
|
||||
:none nil
|
||||
:string (.getRawValue cell)
|
||||
:boolean (.asBoolean cell)
|
||||
(double (.asNumber cell)))))
|
||||
(doubleValue [this]
|
||||
(if (formula-type? (.getType cell))
|
||||
(Double/parseDouble (.getRawValue cell))
|
||||
(double (.asNumber cell))))
|
||||
(boolValue [this] (.asBoolean cell))))
|
||||
|
||||
|
||||
(defn- wrap-row
|
||||
[^Row row]
|
||||
(reify
|
||||
Spreadsheet$Row
|
||||
(getRowNum [this] (dec (.getRowNum row)))
|
||||
(iterator [this]
|
||||
(let [iter (-> (.stream row)
|
||||
(.filter (reify
|
||||
java.util.function.Predicate
|
||||
(test [this val] (not= val nil))))
|
||||
(.iterator))]
|
||||
(reify java.util.Iterator
|
||||
;;In some cases hasNext throws.
|
||||
(hasNext [this] (try (.hasNext iter)
|
||||
(catch Exception _e false)))
|
||||
(next [this] (wrap-cell (.next iter))))))))
|
||||
|
||||
(defn- wrap-sheet
|
||||
[^Sheet sheet]
|
||||
(reify
|
||||
Spreadsheet$Sheet
|
||||
(name [this] (.getName sheet))
|
||||
(id [this] (.getId sheet))
|
||||
(stableId [this] (.getStableId sheet))
|
||||
(iterator [this]
|
||||
(let [iter (.iterator (.openStream sheet))]
|
||||
(reify java.util.Iterator
|
||||
(hasNext [this] (.hasNext iter))
|
||||
(next [this] (wrap-row (.next iter))))))))
|
||||
|
||||
|
||||
(defn input->workbook
|
||||
"Given an input data source, return an implementation of
|
||||
`tech.v3.dataset/Spreadsheet$Workbook`. This interface allows you
|
||||
to iterate through sheets without necessarily parsing them.
|
||||
Once you have a spreadsheet, use `tech.v3.dataset.io.spreadsheet/sheet->dataset`
|
||||
to get a dataset."
|
||||
(^Spreadsheet$Workbook [input]
|
||||
(input->workbook input {}))
|
||||
(^Spreadsheet$Workbook [input _options]
|
||||
(let [workbook (ReadableWorkbook. (io/input-stream input))]
|
||||
(reify
|
||||
Spreadsheet$Workbook
|
||||
(close [this] (.close workbook))
|
||||
(iterator [this]
|
||||
(let [sheet-iter (.iterator (.getSheets workbook))]
|
||||
(reify java.util.Iterator
|
||||
(hasNext [this] (.hasNext sheet-iter))
|
||||
(next [this] (wrap-sheet (.next sheet-iter))))))))))
|
||||
|
||||
|
||||
(defn workbook->datasets
|
||||
"Given a workbook, an string filename or an input stream return a sequence of
|
||||
dataset named after the sheets.
|
||||
|
||||
Options are a subset of the arguments for tech.v3.dataset/->dataset:
|
||||
|
||||
* `:header-row?`
|
||||
* `:num-rows`
|
||||
* `:n-initial-skip-rows`
|
||||
* `:parser-fn`"
|
||||
([input options]
|
||||
(let [workbook (input->workbook input options)]
|
||||
(try
|
||||
(mapv #(parse-spreadsheet/sheet->dataset % options) workbook)
|
||||
(finally
|
||||
(when-not (identical? input workbook)
|
||||
(.close workbook))))))
|
||||
([input]
|
||||
(workbook->datasets input {})))
|
||||
|
||||
|
||||
(defmethod ds-io/data->dataset :xlsx
|
||||
[data options]
|
||||
(let [datasets (workbook->datasets data options)
|
||||
n-datasets (count datasets)]
|
||||
(errors/when-not-errorf
|
||||
(== 1 n-datasets)
|
||||
(if (== 0 n-datasets)
|
||||
"No (%d) datasets found in file"
|
||||
"Multiple (%d) datasets found in file")
|
||||
n-datasets)
|
||||
(first datasets)))
|
||||
@@ -0,0 +1,63 @@
|
||||
(ns tech.v3.libs.guava.cache
|
||||
"Use a google guava cache to memoize function results. Function must not return
|
||||
nil values. Exceptions propagate to caller."
|
||||
(:refer-clojure :exclude [memoize])
|
||||
(:import [com.google.common.cache
|
||||
Cache CacheBuilder CacheLoader
|
||||
LoadingCache CacheStats]
|
||||
[java.time Duration]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
|
||||
(defn memoize
|
||||
"Create a threadsafe, efficient memoized function using a guavacache backing store."
|
||||
[f & {:keys [write-ttl-ms
|
||||
access-ttl-ms
|
||||
soft-values?
|
||||
weak-values?
|
||||
max-size
|
||||
record-stats?]}]
|
||||
(let [^CacheBuilder new-builder
|
||||
(cond-> (CacheBuilder/newBuilder)
|
||||
access-ttl-ms
|
||||
(.expireAfterAccess (Duration/ofMillis access-ttl-ms))
|
||||
write-ttl-ms
|
||||
(.expireAfterWrite (Duration/ofMillis write-ttl-ms))
|
||||
soft-values?
|
||||
(.softValues)
|
||||
weak-values?
|
||||
(.weakValues)
|
||||
max-size
|
||||
(.maximumSize (long max-size))
|
||||
record-stats?
|
||||
(.recordStats))
|
||||
^LoadingCache cache
|
||||
(.build new-builder
|
||||
(proxy [CacheLoader] []
|
||||
(load [args]
|
||||
(if-let [retval (apply f args)]
|
||||
retval
|
||||
(throw (ex-info
|
||||
(format "Nil values not allowed in cache: %s" args)
|
||||
{}))))))]
|
||||
(-> (fn [& args]
|
||||
(.get cache args))
|
||||
(with-meta {:cache cache}))))
|
||||
|
||||
|
||||
(defn memo-stats
|
||||
"Get cache stats from a memoized object."
|
||||
[filecache]
|
||||
(let [^Cache memo-cache (-> (meta filecache)
|
||||
:cache)
|
||||
^CacheStats cache-map (.stats memo-cache)]
|
||||
{:hit-count (.hitCount cache-map)
|
||||
:hit-rate (.hitRate cache-map)
|
||||
:miss-count (.missCount cache-map)
|
||||
:miss-rate (.missRate cache-map)
|
||||
:load-success-count (.loadSuccessCount cache-map)
|
||||
:load-exception-count (.loadExceptionCount cache-map)
|
||||
:average-load-penalty-nanos (.averageLoadPenalty cache-map)
|
||||
:total-load-time-nanos (.totalLoadTime cache-map)
|
||||
:eviction-count (.evictionCount cache-map)}))
|
||||
+1213
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,156 @@
|
||||
(ns tech.v3.libs.parquet-hardwood
|
||||
(:require [ham-fisted.lazy-caching :as lznc]
|
||||
[ham-fisted.api :as hamf]
|
||||
[ham-fisted.iterator :as hamf-iter]
|
||||
[tech.v3.datatype :as dt]
|
||||
[tech.v3.datatype.array-buffer :as array-buffer]
|
||||
[tech.v3.datatype.errors :as errors]
|
||||
[tech.v3.dataset :as ds]
|
||||
[tech.v3.dataset.io.column-parsers :as col-parsers]
|
||||
[tech.v3.dataset.io.context :as parse-context]
|
||||
[tech.v3.dataset.base :as ds-base]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.tools.logging :as log])
|
||||
(:import [dev.morling.hardwood.reader ParquetFileReader ColumnReader]
|
||||
[dev.morling.hardwood.schema FileSchema ColumnSchema]
|
||||
[dev.morling.hardwood.metadata RowGroup ColumnChunk LogicalType
|
||||
LogicalType$DateType LogicalType$StringType LogicalType$UuidType
|
||||
LogicalType$EnumType LogicalType$JsonType LogicalType$TimestampType
|
||||
LogicalType$BsonType LogicalType$IntervalType LogicalType$IntType
|
||||
LogicalType$DecimalType LogicalType$TimeType LogicalType$MapType
|
||||
LogicalType$ListType]
|
||||
[clojure.lang MapEntry]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:private column-finalizers
|
||||
{LogicalType$StringType (fn [_ str-data] (dt/emap (fn [ss] (String. ^bytes ss)) :string str-data))
|
||||
LogicalType$DateType (fn [_ int-data] (-> (dt/as-array-buffer int-data)
|
||||
(array-buffer/set-datatype :packed-local-date)))})
|
||||
|
||||
(defn- read-column
|
||||
[idx ^ColumnSchema col-schema ^ColumnChunk col-def ^ParquetFileReader rdr options]
|
||||
(let [crdr (.getColumnReader rdr col-schema col-def)
|
||||
parser (col-parsers/promotional-object-parser (.name col-schema) nil)
|
||||
lt (.logicalType col-schema)
|
||||
finalizer (get column-finalizers (type lt) (fn [_ v] v))
|
||||
key-fn (:key-fn options identity)]
|
||||
(loop [idx 0]
|
||||
(if (.hasNext crdr)
|
||||
(do
|
||||
(when-let [nv (.readNext crdr)] (col-parsers/add-value! parser idx nv))
|
||||
(recur (inc idx)))
|
||||
{:column-name (key-fn (.name col-schema))
|
||||
:column-parser (reify tech.v3.dataset.io.column_parsers.PParser
|
||||
(finalize [this n-rows]
|
||||
(-> (col-parsers/finalize! parser n-rows)
|
||||
(update :tech.v3.dataset/data #(finalizer lt %)))))
|
||||
:n-rows idx}))))
|
||||
|
||||
(defn- ->path
|
||||
[data]
|
||||
(if (instance? java.nio.file.Path data)
|
||||
data
|
||||
(java.nio.file.Paths/get (str data) (into-array String []))))
|
||||
|
||||
(defn parquet->ds-seq
|
||||
[path options]
|
||||
(let [rdr (ParquetFileReader/open (->path path))
|
||||
schema (.getFileSchema rdr)
|
||||
schema-columns (->> (range (.getColumnCount schema))
|
||||
(into [] (map #(.getColumn schema (long %)))))
|
||||
col-name->idx (into {} (map-indexed (fn [idx ^ColumnSchema col]
|
||||
[(.name col) idx]))
|
||||
schema-columns)
|
||||
col-idx->name (into {} (map-indexed (fn [idx ^ColumnSchema col]
|
||||
[idx (.name col)]))
|
||||
schema-columns)
|
||||
cref->idx (fn [centry] (if (number? centry) (long centry) (col-name->idx centry)))
|
||||
allowset (into #{} (map cref->idx) (get options :column-allowlist))
|
||||
blockset (into #{} (map cref->idx) (get options :column-blocklist))
|
||||
is-allowed? (fn [^long idx]
|
||||
(if-not (empty? allowset)
|
||||
(boolean (allowset idx))
|
||||
(if-not (empty? blockset)
|
||||
(boolean (not (blockset idx)))
|
||||
true)))
|
||||
src-indexes (vec (->> (hamf/range (count schema-columns))
|
||||
(lznc/filter is-allowed?)))
|
||||
rv (->> (.getFileMetaData rdr)
|
||||
(.rowGroups)
|
||||
(lznc/map (fn [^RowGroup grp]
|
||||
(let [parsers (->> src-indexes
|
||||
(hamf/pmap-io 12 (fn [^long idx]
|
||||
(read-column idx (nth schema-columns idx)
|
||||
(nth (.columns grp) idx)
|
||||
rdr
|
||||
options)))
|
||||
(vec))
|
||||
row-count (reduce max 0 (lznc/map :n-rows parsers))]
|
||||
(parse-context/parsers->dataset options parsers row-count)))))
|
||||
closer* (delay (.close rdr))
|
||||
seq* (delay (seq rv))]
|
||||
(reify
|
||||
java.lang.AutoCloseable
|
||||
(close [this] @closer*)
|
||||
clojure.lang.Seqable
|
||||
(seq [this] @seq*)
|
||||
Iterable
|
||||
(iterator [this]
|
||||
(if (realized? seq*)
|
||||
(.iterator ^Iterable @seq*)
|
||||
(let [src-iter (.iterator ^Iterable rv)]
|
||||
(reify
|
||||
java.util.Iterator
|
||||
(hasNext [this] (let [rv (.hasNext src-iter)]
|
||||
(when-not rv @closer*)
|
||||
rv))
|
||||
(next [this] (.next src-iter))))))
|
||||
clojure.lang.IReduceInit
|
||||
(reduce [this rfn acc]
|
||||
(let [acc
|
||||
(if (realized? seq*)
|
||||
(reduce rfn acc @seq*)
|
||||
(reduce rfn acc rv))]
|
||||
@closer*
|
||||
acc)))))
|
||||
|
||||
(defn parquet->ds
|
||||
"Load a parquet file. Input must be a file on disk.
|
||||
|
||||
Options are a subset of the options used for loading datasets -
|
||||
specifically `:column-allowlist` and `:column-blocklist` can be
|
||||
useful here. The parquet metadata ends up as metadata on the
|
||||
datasets."
|
||||
([input options]
|
||||
(let [data-file (io/file input)
|
||||
_ (errors/when-not-errorf
|
||||
(.exists data-file)
|
||||
"Only on-disk files work with parquet. %s does not resolve to a file"
|
||||
input)
|
||||
dataset-seq (vec (parquet->ds-seq (.getCanonicalPath data-file) options))]
|
||||
(when-not (or (:disable-parquet-warn-on-multiple-datasets options)
|
||||
(== 1 (count dataset-seq)))
|
||||
(log/warnf "Concatenating multiple datasets (%d) into one.
|
||||
To disable this warning use `:disable-parquet-warn-on-multiple-datasets`"
|
||||
(count dataset-seq)))
|
||||
(if (== 1 (count dataset-seq))
|
||||
(first dataset-seq)
|
||||
(apply ds-base/concat-copying dataset-seq))))
|
||||
([input]
|
||||
(parquet->ds input nil)))
|
||||
|
||||
;; No writing interface defined yet
|
||||
;; (defn ds-seq->parquet
|
||||
;; ([path options ds-seq])
|
||||
;; ([path ds-seq]))
|
||||
|
||||
;; (defn ds->parquet
|
||||
;; "Write a dataset to a parquet file. Many parquet options are possible;
|
||||
;; these can also be passed in via ds/->write!
|
||||
|
||||
;; Options are the same as ds-seq->parquet."
|
||||
;; ([ds path options]
|
||||
;; (ds-seq->parquet path options [ds]))
|
||||
;; ([ds path]
|
||||
;; (ds->parquet ds path nil)))
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
(ns tech.v3.libs.poi
|
||||
"Parse a dataset in xls or xlsx format. This namespace auto-registers a handler for
|
||||
the `xls` file type so that when using ->dataset, `xls` will automatically map to
|
||||
`(first (workbook->datasets))`.
|
||||
|
||||
Note that this namespace does **not** auto-register a handler for the `xlsx` file
|
||||
type. `xlsx` is handled by the fastexcel namespace.
|
||||
|
||||
If you have an `xlsx` or `xls` file that contains multiple sheets and you want a
|
||||
dataset out of each sheet you have to use `workbook->datasets` as opposed to the
|
||||
higher level `->dataset` operator.
|
||||
|
||||
For serializing datasets to xlsx or xls formats please see
|
||||
[kixi.large](https://github.com/MastodonC/kixi.large).
|
||||
|
||||
Note that poi has many versions and many version conflicts and for instance the
|
||||
docjure library relies on a much older version of poi."
|
||||
(:require [tech.v3.io :as io]
|
||||
[tech.v3.datatype :as dtype]
|
||||
[tech.v3.datatype.protocols :as dtype-proto]
|
||||
[tech.v3.datatype.errors :as errors]
|
||||
[tech.v3.datatype.datetime :as dtype-dt]
|
||||
[tech.v3.dataset.io.spreadsheet :as parse-spreadsheet]
|
||||
[tech.v3.dataset.io :as ds-io])
|
||||
(:import [org.apache.poi.ss.usermodel Workbook Sheet Cell
|
||||
CellType Row]
|
||||
[tech.v3.dataset Spreadsheet$Workbook Spreadsheet$Sheet
|
||||
Spreadsheet$Row Spreadsheet$Cell]
|
||||
[org.apache.poi.xssf.usermodel XSSFWorkbook]
|
||||
[org.apache.poi.hssf.usermodel HSSFWorkbook]
|
||||
[org.apache.poi.ss.usermodel DateUtil]))
|
||||
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
|
||||
(defn- cell-type->keyword
|
||||
[^CellType cell-type]
|
||||
(condp = cell-type
|
||||
CellType/BLANK :none
|
||||
CellType/_NONE :none
|
||||
CellType/NUMERIC :float64
|
||||
CellType/BOOLEAN :boolean
|
||||
CellType/STRING :string))
|
||||
|
||||
|
||||
(defn- wrap-cell
|
||||
[^Cell cell]
|
||||
(reify
|
||||
dtype-proto/PElemwiseDatatype
|
||||
(elemwise-datatype [this]
|
||||
(let [cell-type (.getCellType cell)]
|
||||
(if (or (= cell-type CellType/FORMULA)
|
||||
(= cell-type CellType/ERROR))
|
||||
(cell-type->keyword (.getCachedFormulaResultType cell))
|
||||
(let [dtype-kwd (cell-type->keyword (.getCellType cell))]
|
||||
(if (and (identical? :float64 dtype-kwd)
|
||||
(DateUtil/isCellDateFormatted cell))
|
||||
:local-date
|
||||
dtype-kwd)))))
|
||||
Spreadsheet$Cell
|
||||
(getColumnNum [this] (.. cell getAddress getColumn))
|
||||
(missing [this] (identical? :none (dtype/get-datatype this)))
|
||||
(value [this]
|
||||
(case (dtype-proto/elemwise-datatype this)
|
||||
:none nil
|
||||
:string (.getStringCellValue cell)
|
||||
:boolean (.getBooleanCellValue cell)
|
||||
:local-date (-> (.getLocalDateTimeCellValue cell)
|
||||
(dtype-dt/local-date-time->local-date))
|
||||
(.getNumericCellValue cell)))
|
||||
(doubleValue [this] (.getNumericCellValue cell))
|
||||
(boolValue [this] (.getBooleanCellValue cell))))
|
||||
|
||||
|
||||
(defn- wrap-row
|
||||
^Spreadsheet$Row [^Row row]
|
||||
(reify
|
||||
Spreadsheet$Row
|
||||
(getRowNum [this] (.getRowNum row))
|
||||
(iterator [this]
|
||||
(let [iter (.iterator row)]
|
||||
(reify java.util.Iterator
|
||||
(hasNext [this] (.hasNext iter))
|
||||
(next [this] (wrap-cell (.next iter))))))))
|
||||
|
||||
|
||||
(defn- wrap-sheet
|
||||
^Spreadsheet$Sheet [^Sheet sheet]
|
||||
(reify
|
||||
Spreadsheet$Sheet
|
||||
(name [this] (.getSheetName sheet))
|
||||
(iterator [this]
|
||||
(let [iter (.iterator sheet)]
|
||||
(reify java.util.Iterator
|
||||
(hasNext [this] (.hasNext iter))
|
||||
(next [this] (wrap-row (.next iter))))))))
|
||||
|
||||
|
||||
(defn- fname->file-type
|
||||
[^String fname]
|
||||
(if (.endsWith fname "xls")
|
||||
:xls
|
||||
:xlsx))
|
||||
|
||||
|
||||
|
||||
(defn input->workbook
|
||||
"Given an input data source, return an implementation of
|
||||
`tech.v3.dataset/Spreadsheet$Workbook`. This interface allows you
|
||||
to iterate through sheets without necessarily parsing them.
|
||||
Once you have a spreadsheet, use `tech.v3.dataset.io.spreadsheet/sheet->dataset`
|
||||
to get a dataset."
|
||||
(^Spreadsheet$Workbook [input options]
|
||||
(if (instance? Spreadsheet$Workbook input)
|
||||
input
|
||||
(let [file-type (or (:file-type options)
|
||||
(when (string? input)
|
||||
(fname->file-type input))
|
||||
:xlsx)
|
||||
^Workbook workbook
|
||||
(case file-type
|
||||
:xlsx (XSSFWorkbook. (io/input-stream input))
|
||||
:xls (HSSFWorkbook. (io/input-stream input)))]
|
||||
(reify
|
||||
Spreadsheet$Workbook
|
||||
(iterator [this]
|
||||
(let [iter (.iterator workbook)]
|
||||
(reify java.util.Iterator
|
||||
(hasNext [this] (.hasNext iter))
|
||||
(next [this]
|
||||
(wrap-sheet (.next iter))))))
|
||||
(close [this] (.close workbook))))))
|
||||
(^Spreadsheet$Workbook [input]
|
||||
(input->workbook input {})))
|
||||
|
||||
|
||||
(defn workbook->datasets
|
||||
"Given a workbook, a string filename or an input stream return a sequence of
|
||||
datasets named after the sheets.
|
||||
|
||||
Options are a subset of the arguments for tech.v3.dataset/->dataset:
|
||||
|
||||
* `:file-type` - either `:xls` or `:xlsx` - inferred from the filename when input is
|
||||
a string. If input is a stream defaults to `:xlsx`.
|
||||
* `:header-row?`
|
||||
* `:num-rows`
|
||||
* `:n-initial-skip-rows`
|
||||
* `:parser-fn`"
|
||||
([input options]
|
||||
(let [workbook (input->workbook input options)]
|
||||
(try
|
||||
(mapv #(parse-spreadsheet/sheet->dataset % options) workbook)
|
||||
(finally
|
||||
(when-not (identical? input workbook)
|
||||
(.close workbook))))))
|
||||
([input]
|
||||
(workbook->datasets input {})))
|
||||
|
||||
|
||||
(defmethod ds-io/data->dataset :xls
|
||||
[data options]
|
||||
(let [datasets (workbook->datasets data options)
|
||||
n-datasets (count datasets)]
|
||||
(errors/when-not-errorf
|
||||
(== 1 n-datasets)
|
||||
(if (== 0 n-datasets)
|
||||
"No (%d) datasets found in file"
|
||||
"Multiple (%d) datasets found in file")
|
||||
n-datasets)
|
||||
(first datasets)))
|
||||
+220
@@ -0,0 +1,220 @@
|
||||
(ns tech.v3.libs.spark
|
||||
(:require [tech.v3.dataset.impl.dataset :as ds-impl]
|
||||
[tech.v3.dataset.readers :as ds-readers]
|
||||
[tech.v3.dataset.utils :as ds-utils]
|
||||
[tech.v3.dataset.column :as ds-col]
|
||||
[tech.v3.dataset.io :as ds-io]
|
||||
[tech.v3.datatype.base :as dtype-base]
|
||||
[tech.v3.datatype.emap :as emap]
|
||||
[tech.v3.datatype.datetime :as dtype-dt]
|
||||
[tech.v3.datatype.packing :as packing]
|
||||
[tech.v3.datatype.casting :as casting]
|
||||
[tech.v3.datatype.errors :as errors]
|
||||
[clojure.set :as set])
|
||||
(:import [org.apache.spark.sql Row RowFactory Dataset SparkSession]
|
||||
[org.apache.spark.sql.types StructType StructField
|
||||
DataTypes DataType]
|
||||
[tech.v3.datatype ObjectReader]
|
||||
[java.time LocalDate Instant]
|
||||
[java.util List]))
|
||||
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
|
||||
(def ^:private datatype->sql-map
|
||||
{:boolean DataTypes/BooleanType
|
||||
:int8 DataTypes/ByteType
|
||||
:int16 DataTypes/ShortType
|
||||
:int32 DataTypes/IntegerType
|
||||
:int64 DataTypes/LongType
|
||||
:float32 DataTypes/FloatType
|
||||
:float64 DataTypes/DoubleType
|
||||
:sql-date DataTypes/DateType
|
||||
:string DataTypes/StringType
|
||||
:text DataTypes/StringType
|
||||
:sql-timestamp DataTypes/TimestampType})
|
||||
|
||||
|
||||
(defn datatype->sql-type
|
||||
^DataType [datatype]
|
||||
(let [datatype (packing/unpack-datatype datatype)
|
||||
datatype (if-not (= datatype :epoch-days)
|
||||
(let [temp-dt (casting/un-alias-datatype datatype)]
|
||||
(if (casting/numeric-type? temp-dt)
|
||||
(casting/safe-flatten temp-dt)
|
||||
temp-dt))
|
||||
datatype)
|
||||
sql-type (get datatype->sql-map datatype)
|
||||
_ (errors/when-not-errorf
|
||||
sql-type
|
||||
"Failed to convert datatype %s"
|
||||
datatype)]
|
||||
sql-type))
|
||||
|
||||
|
||||
(def ^:private sql->datatype-map
|
||||
(set/map-invert datatype->sql-map))
|
||||
|
||||
|
||||
(defn sql-type->datatype
|
||||
[sql-type]
|
||||
(if-let [retval (get sql->datatype-map sql-type)]
|
||||
retval
|
||||
(errors/throwf "Failed to find datatype for sql type %s"
|
||||
sql-type)))
|
||||
|
||||
(defn sql-date->local-date
|
||||
^java.time.LocalDate [^java.sql.Date date]
|
||||
(dtype-dt/milliseconds-since-epoch->local-date
|
||||
(.getTime date)))
|
||||
|
||||
|
||||
(defn local-date->sql-date
|
||||
^java.sql.Date [^LocalDate ld]
|
||||
(java.sql.Date/valueOf ld))
|
||||
|
||||
|
||||
(casting/add-object-datatype! :sql-date java.sql.Date)
|
||||
|
||||
|
||||
(defn sql-timestamp->instant
|
||||
^Instant [^java.sql.Timestamp date]
|
||||
(dtype-dt/milliseconds-since-epoch->instant
|
||||
(.getTime date)))
|
||||
|
||||
|
||||
(defn instant->sql-timestamp
|
||||
^java.sql.Timestamp [^Instant inst]
|
||||
(java.sql.Timestamp.
|
||||
(dtype-dt/instant->milliseconds-since-epoch inst)))
|
||||
|
||||
|
||||
(casting/add-object-datatype! :sql-timestamp java.sql.Timestamp)
|
||||
|
||||
|
||||
(defn- prepare-ds-for-spark
|
||||
[ds]
|
||||
(reduce
|
||||
(fn [ds col]
|
||||
(let [cname (ds-utils/column-safe-name (ds-col/column-name col))
|
||||
col-dtype (packing/unpack-datatype
|
||||
(dtype-base/elemwise-datatype col))
|
||||
missing (ds-col/missing col)
|
||||
epoch-types #{:epoch-milliseconds :epoch-seconds
|
||||
:epoch-microseconds}
|
||||
coldata (cond
|
||||
(epoch-types col-dtype)
|
||||
(->> (dtype-dt/epoch->datetime :instant col)
|
||||
(emap/emap instant->sql-timestamp :sql-timestamp))
|
||||
(= col-dtype :local-date)
|
||||
(emap/emap local-date->sql-date :sql-date
|
||||
(packing/unpack col))
|
||||
(= col-dtype :epoch-days)
|
||||
(->> (dtype-dt/epoch->datetime :local-date col)
|
||||
(emap/emap instant->sql-timestamp :sql-date))
|
||||
(= col-dtype :text)
|
||||
(emap/emap str :string col)
|
||||
(and (casting/numeric-type? col-dtype)
|
||||
(not= col-dtype (casting/safe-flatten col-dtype)))
|
||||
(dtype-base/elemwise-cast col (casting/safe-flatten col-dtype))
|
||||
:else
|
||||
col)]
|
||||
(assoc ds cname (ds-col/new-column cname coldata (meta col) missing))))
|
||||
ds
|
||||
(vals ds)))
|
||||
|
||||
|
||||
(defn ds-schema->spark-schema
|
||||
^StructType [ds-schema]
|
||||
(let [retval (StructType.)]
|
||||
(reduce
|
||||
(fn [retval col]
|
||||
(let [{:keys [datatype name]} col
|
||||
name (ds-utils/column-safe-name name)
|
||||
nullable? (boolean (not (.isEmpty (ds-col/missing col))))]
|
||||
(.add ^StructType retval name (datatype->sql-type datatype) nullable?)))
|
||||
retval
|
||||
(:columns ds-schema))))
|
||||
|
||||
|
||||
(defn ds-schema
|
||||
[ds]
|
||||
(assoc (meta ds)
|
||||
:columns (mapv meta (vals ds))))
|
||||
|
||||
|
||||
(defn- dataset->row-list
|
||||
^List [ds]
|
||||
(let [val-rdr (ds-readers/value-reader ds)
|
||||
n-elems (.lsize val-rdr)]
|
||||
(reify ObjectReader
|
||||
(lsize [rdr] n-elems)
|
||||
(readObject [rdr idx]
|
||||
(RowFactory/create
|
||||
(object-array (.readObject val-rdr idx)))))))
|
||||
|
||||
|
||||
(defn ds->spark-dataset
|
||||
(^Dataset [ds ^SparkSession spark-session _options]
|
||||
;;Prepare the dataset datatypes
|
||||
(let [ds (prepare-ds-for-spark ds)]
|
||||
(.createDataFrame spark-session (dataset->row-list ds)
|
||||
(-> (ds-schema ds)
|
||||
(ds-schema->spark-schema)))))
|
||||
(^Dataset [ds session]
|
||||
(ds->spark-dataset ds session nil)))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
(defn collect-spark-dataset->ds
|
||||
[^Dataset dataset]
|
||||
;;Make sure we are dealing with rows, not drama
|
||||
;;This system will not be able to deal with missing
|
||||
;;values without scanning each column.
|
||||
(let [dataset (.toDF dataset)
|
||||
schema (.schema dataset)
|
||||
;;data is an array of rows
|
||||
^objects data (.collect dataset)
|
||||
n-rows (alength data)]
|
||||
(->> (.fields schema)
|
||||
(map-indexed
|
||||
(fn [^long col-idx ^StructField field]
|
||||
(let [colname (.name field)
|
||||
dtype (sql-type->datatype (.dataType field))]
|
||||
(ds-col/new-column colname
|
||||
(reify ObjectReader
|
||||
(elemwiseDatatype [rdr] dtype)
|
||||
(lsize [rdr] n-rows)
|
||||
(readObject [rdr row-idx]
|
||||
(let [^Row row (aget data row-idx)]
|
||||
(.get row col-idx))))
|
||||
nil
|
||||
[]))))
|
||||
(ds-impl/new-dataset))))
|
||||
|
||||
|
||||
|
||||
(comment
|
||||
;; databricks-connect specific classes
|
||||
;; should work similar for spark-connect
|
||||
|
||||
;;Tested with hese deps
|
||||
;;org.scala-lang/scala-reflect {:mvn/version "2.12.18"}
|
||||
;;com.databricks/databricks-connect {:mvn/version "16.1.0"}
|
||||
|
||||
|
||||
(import
|
||||
'[com.databricks.connect DatabricksSession]
|
||||
'[com.databricks.sdk.core DatabricksConfig])
|
||||
|
||||
(def config (.. (DatabricksConfig.) (setProfile "adb-xxxxx")))
|
||||
(def spark (.. (DatabricksSession/builder) (sdkConfig config) getOrCreate))
|
||||
|
||||
(->
|
||||
(.sql spark "show catalogs;")
|
||||
collect-spark-dataset->ds)
|
||||
|
||||
)
|
||||
+326
@@ -0,0 +1,326 @@
|
||||
(ns tech.v3.libs.tribuo
|
||||
"Bindings to make working with tribuo more straight forward when using datasets.
|
||||
|
||||
```clojure
|
||||
;; Classification
|
||||
|
||||
tech.v3.dataset.tribuo-test> (def ds (classification-example-ds 10000))
|
||||
#'tech.v3.dataset.tribuo-test/ds
|
||||
tech.v3.dataset.tribuo-test> (def model (tribuo/train-classification (org.tribuo.classification.xgboost.XGBoostClassificationTrainer. 6) ds :label))
|
||||
#'tech.v3.dataset.tribuo-test/model
|
||||
tech.v3.dataset.tribuo-test> (ds/head (tribuo/predict-classification model (ds/remove-columns ds [:label])))
|
||||
_unnamed [5 3]:
|
||||
|
||||
| :prediction | red | green |
|
||||
|-------------|-----------:|-----------:|
|
||||
| red | 0.92524981 | 0.07475022 |
|
||||
| green | 0.07464883 | 0.92535114 |
|
||||
| green | 0.07464883 | 0.92535114 |
|
||||
| red | 0.92525917 | 0.07474083 |
|
||||
| green | 0.07464883 | 0.92535114 |
|
||||
|
||||
|
||||
;; Regression
|
||||
tech.v3.dataset.tribuo-test> (def ds (ds/->dataset \"test/data/winequality-red.csv\" {:separator \\;}))
|
||||
#'tech.v3.dataset.tribuo-test/ds
|
||||
tech.v3.dataset.tribuo-test> (def model (tribuo/train-regression (org.tribuo.regression.xgboost.XGBoostRegressionTrainer. 50) ds \"quality\"))
|
||||
#'tech.v3.dataset.tribuo-test/model
|
||||
tech.v3.dataset.tribuo-test> (ds/head (tribuo/predict-regression model (ds/remove-columns ds [\"quality\"])))
|
||||
_unnamed [5 1]:
|
||||
|
||||
| :prediction |
|
||||
|------------:|
|
||||
| 5.01974726 |
|
||||
| 5.02164841 |
|
||||
| 5.22696543 |
|
||||
| 5.79519272 |
|
||||
| 5.01974726 |
|
||||
```"
|
||||
(:require [tech.v3.dataset :as ds]
|
||||
[tech.v3.dataset.modelling :as modelling]
|
||||
[tech.v3.datatype :as dtype]
|
||||
[tech.v3.datatype.casting :as casting]
|
||||
[clojure.set :as set]
|
||||
[charred.api :as charred])
|
||||
(:import [tech.v3.datatype Buffer ArrayHelpers]
|
||||
[java.util HashMap Map List]
|
||||
[java.nio.file Files]
|
||||
[java.nio.file.attribute FileAttribute]
|
||||
[org.tribuo.classification Label LabelFactory]
|
||||
[org.tribuo DataSource Output OutputFactory Trainer Model MutableDataset
|
||||
Prediction]
|
||||
[org.tribuo.impl ArrayExample]
|
||||
[org.tribuo.provenance SimpleDataSourceProvenance]
|
||||
[org.tribuo.regression RegressionFactory Regressor]
|
||||
[org.tribuo.regression.evaluation RegressionEvaluator RegressionEvaluation]
|
||||
[com.oracle.labs.mlrg.olcut.config ConfigurationManager]
|
||||
[com.oracle.labs.mlrg.olcut.config.json JsonConfigFactory]))
|
||||
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(ConfigurationManager/addFileFormatFactory (JsonConfigFactory.)) ; allows json config
|
||||
|
||||
(def ^:private str-ary-cls (Class/forName "[Ljava.lang.String;"))
|
||||
(def ^:private dbl-ary-cls (Class/forName "[D"))
|
||||
|
||||
(defn- ->string
|
||||
^String [obj]
|
||||
(if (nil? obj)
|
||||
""
|
||||
(if (or (keyword? obj)
|
||||
(symbol? obj))
|
||||
(name obj)
|
||||
(str obj))))
|
||||
|
||||
(defn- ->string-array
|
||||
^"[Ljava.lang.String;" [data]
|
||||
(if (instance? str-ary-cls data)
|
||||
data
|
||||
(into-array String (map ->string data))))
|
||||
|
||||
(defn- ->double-array
|
||||
"Slightly faster than (double-array) when the source data implements tech.v3.datatype.Buffer."
|
||||
^doubles [data]
|
||||
(if (instance? dbl-ary-cls data)
|
||||
data
|
||||
(let [data (dtype/->buffer data)
|
||||
ndata (.size data)
|
||||
ddata (double-array ndata)]
|
||||
;;Clojure's aset function returns a value and thus boxes every item in the array.
|
||||
(dotimes [idx ndata] (ArrayHelpers/aset ddata idx (.readDouble data idx)))
|
||||
ddata)))
|
||||
|
||||
(defn- ds->regression-outputs
|
||||
[ds]
|
||||
(let [fact (RegressionFactory.)]
|
||||
(with-meta
|
||||
(if (integer? ds)
|
||||
(dtype/make-reader :object ds (.getUnknownOutput fact))
|
||||
(let [rvecs (ds/rowvecs ds)
|
||||
names (->string-array (ds/column-names ds))]
|
||||
(dtype/make-reader :object (dtype/ecount rvecs)
|
||||
(Regressor. names (->double-array (rvecs idx))))))
|
||||
{:output-factory fact})))
|
||||
|
||||
|
||||
(defn- ds->single-label-outputs
|
||||
[ds]
|
||||
(let [label-fact (LabelFactory.)]
|
||||
(if (integer? ds)
|
||||
(with-meta
|
||||
(dtype/make-reader :object ds (.getUnknownOutput label-fact))
|
||||
{:output-factory label-fact})
|
||||
(do
|
||||
(when-not (== 1 (ds/column-count ds))
|
||||
(throw (Exception. "Single label output expected, got multi-label dataset")))
|
||||
(let [col (first (vals ds))
|
||||
colmeta (meta col)
|
||||
data (dtype/->buffer col)
|
||||
n-rows (.lsize data)
|
||||
data (if-let [cat-map (get-in colmeta [:categorical-map :lookup-table])]
|
||||
(let [rev-map (set/map-invert cat-map)]
|
||||
(dtype/make-reader :object n-rows (get rev-map (long (data idx)))))
|
||||
data)]
|
||||
(with-meta
|
||||
(dtype/make-reader :object n-rows (Label. (->string (data idx))))
|
||||
{:output-factory label-fact}))))))
|
||||
|
||||
(defn- ensure-numeric-columns
|
||||
[ds]
|
||||
(when-not (every? casting/numeric-type? (map (comp :datatype meta) (vals ds)))
|
||||
(throw (Exception.
|
||||
(str "All columns must be numeric at this point. Non numeric columns:\n"
|
||||
(->> (map meta (vals ds))
|
||||
(filter #(not (casting/numeric-type? (get % :datatype))))
|
||||
(mapv :name))))))
|
||||
ds)
|
||||
|
||||
(defn- ds->examples
|
||||
^Buffer [ds ds->outputs]
|
||||
(let [inf-columns (modelling/inference-target-column-names ds)
|
||||
inf-ds (when inf-columns (ds/select-columns ds inf-columns))
|
||||
feat-ds (-> (ds/remove-columns ds inf-columns)
|
||||
(ensure-numeric-columns))
|
||||
feat-data (ds/rowvecs feat-ds)
|
||||
n-rows (ds/row-count ds)
|
||||
;;column names have to be strings.
|
||||
cnames (->string-array (ds/column-names feat-ds))
|
||||
n-rows (ds/row-count ds)
|
||||
outputs (if inf-ds
|
||||
(ds->outputs inf-ds)
|
||||
(ds->outputs n-rows))]
|
||||
(with-meta
|
||||
(dtype/make-reader :object n-rows
|
||||
;;idx is a compile-time variable below
|
||||
(ArrayExample. ^Output (outputs idx)
|
||||
cnames (->double-array (feat-data idx))))
|
||||
(meta outputs))))
|
||||
|
||||
(defn- safe-str
|
||||
[n]
|
||||
(cond (string? n)
|
||||
n
|
||||
(or (keyword? n) (symbol? n))
|
||||
(if-let [nn (namespace n)]
|
||||
(str nn "/" (name n))
|
||||
(str (name n)))))
|
||||
|
||||
|
||||
(defn- ds->datasource
|
||||
^DataSource [ds ds->outputs]
|
||||
(let [examples (ds->examples ds ds->outputs)
|
||||
{:keys [output-factory provenance]} (meta examples)
|
||||
provenance (or provenance
|
||||
(SimpleDataSourceProvenance. (safe-str (:name (meta ds))) output-factory))]
|
||||
(when-not output-factory
|
||||
(throw (RuntimeException. "Output factory not present in example metadata")))
|
||||
(reify DataSource
|
||||
(getProvenance [this] provenance)
|
||||
(iterator [this] (.iterator examples))
|
||||
(getOutputFactory [this] output-factory))))
|
||||
|
||||
|
||||
(defn- label-score
|
||||
^double [^Label label]
|
||||
(.getScore label))
|
||||
|
||||
(defn- label-label
|
||||
^String [^Label label]
|
||||
(.getLabel label))
|
||||
|
||||
(defn- prediction-label
|
||||
^String [^Prediction pred]
|
||||
(-> (.getOutput pred)
|
||||
(label-label)))
|
||||
|
||||
(defn- has-probabilities?
|
||||
[^Prediction pred]
|
||||
(.hasProbabilities pred))
|
||||
|
||||
(defn- prediction-scores
|
||||
^Map [^Prediction pred]
|
||||
(.getOutputScores pred))
|
||||
|
||||
|
||||
(defn- regressor-value
|
||||
^double [^Regressor reg]
|
||||
(aget (.getValues reg) 0))
|
||||
|
||||
|
||||
(defn- prediction-regval
|
||||
[^Prediction pred]
|
||||
(regressor-value (.getOutput pred)))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Begin public API
|
||||
|
||||
|
||||
;; Classification
|
||||
|
||||
(defn make-classification-datasource
|
||||
"Make a single label classification datasource."
|
||||
(^DataSource [ds]
|
||||
(make-classification-datasource ds nil))
|
||||
(^DataSource [ds inf-col-name]
|
||||
(if inf-col-name
|
||||
(-> (modelling/set-inference-target ds inf-col-name)
|
||||
(ds->datasource ds->single-label-outputs))
|
||||
(ds->datasource ds ds->single-label-outputs))))
|
||||
|
||||
|
||||
(defn train-classification
|
||||
"Train a single label classification model. Returns the model."
|
||||
^Model [^Trainer trainer ds & [inf-col-name]]
|
||||
(.train trainer (-> (make-classification-datasource ds inf-col-name)
|
||||
(MutableDataset.))))
|
||||
|
||||
|
||||
(defn classification-predictions->dataset
|
||||
"Given the list of predictions from a classification model return a dataset
|
||||
that will include probability distributions when possible. The actual prediction
|
||||
will be in the `:prediction` column."
|
||||
[^List predictions]
|
||||
(let [labels (dtype/emap prediction-label :string predictions)
|
||||
pred (.get predictions 0)]
|
||||
(-> (ds/->dataset
|
||||
(merge {:prediction labels}
|
||||
(when (has-probabilities? pred)
|
||||
(let [scores (prediction-scores pred)]
|
||||
(->> (.entrySet scores)
|
||||
(map (fn [entry]
|
||||
(let [ekey (key entry)]
|
||||
[ekey (-> (dtype/emap #(-> (prediction-scores %)
|
||||
(.get ekey)
|
||||
(label-score))
|
||||
:float64
|
||||
predictions)
|
||||
(dtype/clone))])))
|
||||
(into {}))))))
|
||||
(modelling/set-inference-target :prediction))))
|
||||
|
||||
|
||||
(defn predict-classification
|
||||
"Use this model to predict every row of this dataset returning a new dataset containing
|
||||
at least a `:prediction` column. If this classifier is capable of predicting probability
|
||||
distributions those will be returned as per-label as separate columns."
|
||||
[^Model model ds]
|
||||
(when-not (== 0 (ds/row-count ds))
|
||||
(->> (.predict model (make-classification-datasource ds))
|
||||
(classification-predictions->dataset))))
|
||||
|
||||
|
||||
;; Regression
|
||||
(defn make-regression-datasource
|
||||
"Make a regression datasource from a dataset."
|
||||
(^DataSource [ds]
|
||||
(make-regression-datasource ds nil))
|
||||
(^DataSource [ds inf-col-name]
|
||||
(if inf-col-name
|
||||
(-> (modelling/set-inference-target ds inf-col-name)
|
||||
(ds->datasource ds->regression-outputs))
|
||||
(ds->datasource ds ds->regression-outputs))))
|
||||
|
||||
|
||||
(defn train-regression
|
||||
"Train a regression model on a dataset returning the model."
|
||||
^Model [^Trainer trainer ds & [inf-col-name]]
|
||||
(.train trainer (-> (make-regression-datasource ds inf-col-name)
|
||||
(MutableDataset.))))
|
||||
|
||||
|
||||
(defn predict-regression
|
||||
"Use a regression model to predict each column of the dataset returning a dataset with
|
||||
at least one column named `:prediction`."
|
||||
[^Model model ds]
|
||||
(when-not (== 0 (ds/row-count ds))
|
||||
(ds/->dataset {:prediction (->> (.predict model (make-regression-datasource ds))
|
||||
(dtype/emap prediction-regval :float64)
|
||||
(dtype/clone))})))
|
||||
|
||||
|
||||
(defn evaluate-regression
|
||||
"Evaluate a regression model against this model. Returns map of
|
||||
`{:rmse :mae :r2}`"
|
||||
[^Model model ds inf-col-name]
|
||||
(let [inf-col-name (->string inf-col-name)
|
||||
^RegressionEvaluation eval (.evaluate (RegressionEvaluator.) model
|
||||
(MutableDataset. (make-regression-datasource ds inf-col-name)))
|
||||
dim (Regressor. inf-col-name Double/NaN)]
|
||||
{:rmse (.rmse eval dim)
|
||||
:mae (.mae eval dim)
|
||||
:r2 (.r2 eval dim)}))
|
||||
|
||||
|
||||
|
||||
(defn trainer
|
||||
"Creates a tribuo trainer from a list of config components
|
||||
follwing OLCUT convention. One of the components should be a trainer,
|
||||
which is the looked-up by `trainer-name` and returned."
|
||||
|
||||
[config-components trainer-name]
|
||||
(let [config {:config {:components config-components}}
|
||||
tmp (.toString (Files/createTempFile "config" ".json" (make-array FileAttribute 0)))
|
||||
_ (charred/write-json tmp config)
|
||||
config-manager (ConfigurationManager. tmp)]
|
||||
(.lookup config-manager trainer-name)))
|
||||
Reference in New Issue
Block a user