ajet/df-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init research

Browse Source
This commit is contained in:
Adam Jeniski
2026-02-08 11:20:43 -10:00
commit bdf064f54d
3041 changed files with 1592200 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tablecloth/test/tablecloth/api/aggregate_test.clj
Vendored
+30
View File
@@ -0,0 +1,30 @@
(ns tablecloth.api.aggregate-test
(:require [tablecloth.api :as api]
[tech.v3.datatype.functional :refer [sum]]
[midje.sweet :refer [fact =>]]))
(fact "aggregate keeps an order of columns"
(-> (array-map
:a [1 1]
:b 2
:c 3
:d 4
:e 5
:f 6
:g 7
:h 8
:i 9)
(api/dataset)
(api/aggregate-columns [:a :b :c :d :e :f :g :h :i] sum)
(api/column-names)) => '(:a :b :c :d :e :f :g :h :i))
(fact "aggregate default prefix changed to custom"
(-> (api/dataset {:l [:x :x :y :y :y]
:a [1 2 3 4 5]
:b [5 5 5 5 5]})
(api/group-by :l)
(api/aggregate (fn [ds] {:sum-of-b (reduce + (ds :b))})
{:default-column-name-prefix "xxx"})
(api/column-names)
(set)) => #{:xxx-sum-of-b :$group-name})
tablecloth/test/tablecloth/api/columns_test.clj
Vendored
+74
View File
@@ -0,0 +1,74 @@
(ns tablecloth.api.columns-test
(:require [tablecloth.api :as api]
[tech.v3.datatype :as dtype]
[midje.sweet :refer [tabular fact =>]]))
;; https://github.com/scicloj/tablecloth/issues/9
(def dss (api/dataset {:idx [1 1 1 2 2 2 3 3 3]
:a ["a" "b" "c" "a" "b" "c" "a" "b" "c"]
"z" 1
:b [1 2 3 2 3 4 3 2 1]
:c [3 1 2 4 2 1 3 2 4]}))
(fact "reorder-columns"
(tabular (fact (-> dss
(api/reorder-columns ?order)
(api/column-names))
=>
?expected)
?order ?expected
[:ids :b :a :c] [:b :a :c :idx "z"]
[:idx :b :a "z" :c] [:idx :b :a "z" :c]
[:idx :b :a :C] [:idx :b :a "z" :c]
[:c :A :b :e :z :idx] [:c :b :idx :a "z"]
string? ["z" :idx :a :b :c]
#".*[az]$" [:a "z" :idx :b :c])
(fact [:b :a :c :idx "z"]
=>
(-> dss
(api/reorder-columns :b :a [:c :ids])
(api/column-names))))
(fact "add-or-replace"
(tabular (fact (-> {:x [1 2]}
(api/dataset)
(api/add-or-replace-column :y ?v)
:y
(dtype/get-datatype))
=>
?expected)
?expected ?v
:int64 1
:float64 1.0
:string "abc"))
(fact "add"
(tabular (fact (-> {:x [1 2]}
(api/dataset)
(api/add-column :y ?v)
:y
(dtype/get-datatype))
=>
?expected)
?expected ?v
:int64 1
:float64 1.0
:string "abc"))
(fact "update-columns"
(->
(api/update-columns dss
{:a (fn [row] (map count row))
:b (fn [row] (map inc row))})
(api/select-columns [:a :b])
(api/rows :as-maps)) =>
[{:a 1, :b 2} {:a 1, :b 3} {:a 1, :b 4} {:a 1, :b 3} {:a 1, :b 4} {:a 1, :b 5} {:a 1, :b 4} {:a 1, :b 3} {:a 1, :b 2}]
(->
(api/update-columns dss [:a]
[(fn [row] (map count row))])
:a
seq) => [1 1 1 1 1 1 1 1 1])
tablecloth/test/tablecloth/api/dataset_test.clj
Vendored
+137
View File
@@ -0,0 +1,137 @@
(ns tablecloth.api.dataset-test
(:require [tablecloth.api :as api]
[tablecloth.column.api :as tcc]
[tablecloth.common-test :refer [DS]]
[clojure.java.io :as io]
[midje.sweet :refer [tabular fact => throws]])
(:import [java.io FileNotFoundException]))
(fact "dataset?"
(fact (api/dataset? (api/dataset)) => true)
(fact (api/dataset? DS) => true)
(fact (api/dataset? {}) => false))
(fact "empty-ds"
(fact (api/empty-ds? (api/dataset)) => true)
(fact (api/empty-ds? (api/dataset {:a []})) => true)
(fact (api/empty-ds? DS) => false))
(fact "dataset-creation"
(fact (and (zero? (api/row-count (api/dataset)))
(zero? (api/column-count (api/dataset)))) => true)
(fact (:$value (api/dataset 999))
=> [999])
(fact (get (api/dataset 999 {:single-value-column-name "my-single-value"}) "my-single-value")
=> [999])
(fact ((juxt #(get % 0) api/dataset-name)
(api/dataset 999 {:single-value-column-name ""
:dataset-name "Single value"}))
=> [[999] "Single value"])
(tabular (fact (-> (api/dataset ?input)
(api/columns :as-map))
=> ?res)
?res ?input
{0 [:A :B :C] 1 [33 5 :a]} [[:A 33] [:B 5] [:C :a]]
{0 [:A :B :C] 1 [[1 2 3 4 5 6]
"X"
:a]} [[:A [1 2 3 4 5 6]] [:B "X"] [:C :a]]
{:A [33]} {:A 33}
{:A [1 2 3]} {:A [1 2 3]}
{:A [3 4 5] :B (repeat 3 "X")} {:A [3 4 5] :B "X"}
{:a [1 99] :b [3 2]} [{:a 1 :b 3} {:b 2 :a 99}]
{:a [1 2] :b [[1 2 3] [3 4]]} [{:a 1 :b [1 2 3]} {:a 2 :b [3 4]}]
{:a [nil 3 11] :b [1 4 nil]} [{:a nil :b 1} {:a 3 :b 4} {:a 11}]
{0 [1 3 5] 1 [2 4 6]} [[1 2] [3 4] [5 6]]
{0 [1 4 7]
1 [2 5 8]
2 [3 6 9]} [[1 2 3] [4 5 6] [7 8 9]]
{0 ["a" "b" "c"] 1 [1 2 3]} [["a" 1] ["b" 2] ["c" 3]])
(fact (-> (api/dataset "data/family.csv")
(api/shape))
=> [5 5])
(fact (-> (api/dataset "https://vega.github.io/vega-lite/examples/data/seattle-weather.csv")
(api/shape))
=> [1461 6])
(fact (api/dataset "not-existing.csv")
=> (throws FileNotFoundException)))
(fact "saving"
(fact (do (api/write! DS "DS.tsv.gz")
(.exists (io/file "DS.tsv.gz")))
=> true)
(fact (-> (api/dataset "DS.tsv.gz")
(api/dataset?))
=> true)
(fact (do (api/write! DS "DS.csv.gz")
(.exists (io/file "DS.csv.gz")))
=> true)
(fact (-> (api/dataset "DS.csv.gz")
(api/dataset?))
=> true)
(fact (do (api/write-nippy! DS "DS.nippy.gz")
(.exists (io/file "DS.nippy.gz")))
=> true)
(fact (-> (api/read-nippy "DS.nippy.gz")
(api/dataset?))
=> true))
(fact "dataset-shape"
(fact (api/shape DS)
=> [9 4])
(fact (api/row-count DS)
=> 9)
(fact (api/column-count DS)
=> 4))
(fact "dataset-name"
(fact (api/dataset-name DS)
=> "DS")
(fact (-> [[1 2 3] [4 5 6]]
(api/dataset {:dataset-name "Test"})
(api/dataset-name)) => "Test"))
(fact "dataset-info"
(fact
(-> (api/info DS)
(select-keys [:col-name :datatype]))
=> {:col-name [:V1 :V2 :V3 :V4]
:datatype [:int64 :int64 :float64 :string]})
(fact
(-> (api/info DS)
(api/column-names))
=> '(:col-name :datatype :n-valid :n-missing :min :mean :mode :max :standard-deviation :skew :first :last))
(fact
(-> (api/info DS :basic)
(api/rows :as-maps))
=> [{:name "DS", :columns 4, :rows 9, :grouped? false}])
(fact
(-> (api/info DS :columns)
(api/rows :as-maps))
=> [{:name :V1, :n-elems 9, :categorical? nil, :datatype :int64}
{:name :V2, :n-elems 9, :categorical? nil, :datatype :int64}
{:name :V3, :n-elems 9, :categorical? nil, :datatype :float64}
{:name :V4, :n-elems 9, :categorical? true, :datatype :string}])
(fact
(-> (api/info DS :columns)
(api/rows :as-maps {:nil-missing? false}))
=> [{:name :V1, :n-elems 9, :datatype :int64}
{:name :V2, :n-elems 9, :datatype :int64}
{:name :V3, :n-elems 9, :datatype :float64}
{:name :V4, :n-elems 9, :categorical? true, :datatype :string}]))
(fact "as-double-arrays"
(tabular (fact (-> (api/dataset {:a [1 2 3]
:b [5 6 7]})
(?f :as-double-arrays)
(->> (map seq)))
= ?v)
?f ?v
api/rows '((1.0 5.0) (2.0 6.0) (3.0 7.0))
api/columns '((1.0 2.0 3.0) (5.0 6.0 7.0))))
(fact "let-dataset"
(fact (api/let-dataset [x (range 4) y 10 z (tcc/+ x y)])
=> (api/dataset {:x [0 1 2 3]
:y [10 10 10 10]
:z [10 11 12 13]})))
tablecloth/test/tablecloth/api/fold_unroll_test.clj
Vendored
+18
View File
@@ -0,0 +1,18 @@
(ns tablecloth.api.fold-unroll-test
(:require [tablecloth.api :as api]
[midje.sweet :refer [tabular fact =>]]))
(fact "one-row-ds"
(-> (api/dataset {:a [1] :b [2]})
(api/fold-by :a set)
(api/rows))
=>
[[1 #{2}]])
(fact "empty-ds"
(-> (api/dataset {:a [1]})
(api/drop-rows 0)
(api/fold-by :a)
(api/empty-ds?))
=>
true)
tablecloth/test/tablecloth/api/group_by_test.clj
Vendored
+4
View File
@@ -0,0 +1,4 @@
(ns tablecloth.api.group-by-test
(:require [tablecloth.api.group-by :as sut]
[clojure.test :as t]))
tablecloth/test/tablecloth/api/join_concat_ds_test.clj
Vendored
+113
View File
@@ -0,0 +1,113 @@
(ns tablecloth.api.join-concat-ds-test
(:require [midje.sweet :as midje :refer [fact tabular =>]]
[tablecloth.api :as api]))
(def ds1 (api/dataset {:a [1 2 1 2 3 4 nil nil 4]
:b (range 101 110)
:c (map str "abs tract")}))
(def ds2 (api/dataset {:a [nil 1 2 5 4 3 2 1 nil]
:b (range 110 101 -1)
:c (map str "datatable")
:d (symbol "X")
:e [3 4 5 6 7 nil 8 1 1]}))
(fact "anti-join"
(-> (api/anti-join ds1 ds2 :a)
(api/rows :as-maps)) => []
(-> (api/anti-join ds2 ds1 :a)
(api/rows :as-maps)) => [{:a 5, :b 107, :c "a", :d 'X, :e 6}]
(-> (api/anti-join ds1 ds2 :b)
(api/rows :as-maps)) => [{:b 101, :a 1, :c "a"}]
(-> (api/anti-join ds2 ds1 :b)
(api/rows :as-maps)) => [{:b 110, :a nil, :c "d", :d 'X, :e 3}]
(-> (api/anti-join ds1 ds2 :c)
(api/rows :as-maps)) => [{:c "s", :a 1, :b 103}
{:c " ", :a 2, :b 104}
{:c "r", :a 4, :b 106}
{:c "c", :a nil, :b 108}]
(-> (api/anti-join ds2 ds1 :c)
(api/rows :as-maps)) => [{:c "d", :a nil, :b 110, :d 'X, :e 3}
{:c "l", :a 1, :b 103, :d 'X, :e 1}
{:c "e", :a nil, :b 102, :d 'X, :e 1}]
(-> (api/anti-join ds1 ds2 {:left :a :right :e})
(api/rows :as-maps)) => [{:a 2, :b 102, :c "b"} {:a 2, :b 104, :c " "}]
(-> (api/anti-join ds2 ds1 {:left :e :right :a})
(api/rows :as-maps)) => [{:e 5, :a 2, :b 108, :c "t", :d 'X}
{:e 6, :a 5, :b 107, :c "a", :d 'X}
{:e 7, :a 4, :b 106, :c "t", :d 'X}
{:e 8, :a 2, :b 104, :c "b", :d 'X}]
(-> (api/anti-join ds1 ds2 [:a :b])
(api/rows :as-maps)) => [{:a 1, :b 101, :c "a"}
{:a 2, :b 102, :c "b"}
{:a nil, :b 107, :c "a"}
{:a nil, :b 108, :c "c"}
{:a 4, :b 109, :c "t"}]
(-> (api/anti-join ds2 ds1 [:a :b])
(api/rows :as-maps)) => [{:a nil, :b 110, :c "d", :d 'X, :e 3}
{:a 1, :b 109, :c "a", :d 'X, :e 4}
{:a 2, :b 108, :c "t", :d 'X, :e 5}
{:a 5, :b 107, :c "a", :d 'X, :e 6}
{:a nil, :b 102, :c "e", :d 'X, :e 1}])
(fact "semi-join"
(-> (api/semi-join ds1 ds2 :a)
(api/row-count)) => 9
(-> (api/semi-join ds2 ds1 :a)
(api/row-count)) => 8
(-> (api/semi-join ds1 ds2 :b)
(api/row-count)) => 8
(-> (api/semi-join ds2 ds1 :b)
(api/row-count)) => 8
(-> (api/semi-join ds1 ds2 :c)
(api/row-count)) => 5
(-> (api/semi-join ds2 ds1 :c)
(api/row-count)) => 6
(-> (api/semi-join ds1 ds2 {:left :a :right :e})
(api/row-count)) => 7
(-> (api/semi-join ds2 ds1 {:left :e :right :a})
(api/row-count)) => 5
(-> (api/semi-join ds1 ds2 [:a :b])
(api/row-count)) => 4
(-> (api/semi-join ds2 ds1 [:a :b])
(api/row-count)) => 4)
(fact "eraderna int-string join"
(-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}]))
(-> (api/dataset [{:i "foo" :y 2022 :s "2022"}
{:i "foo" :y 2023 :s "2023"}]))
[:i :y])
(api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}]
(-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}])
(api/convert-types {:y :int16}))
(-> (api/dataset [{:i "foo" :y 2022 :s "2022"}
{:i "foo" :y 2023 :s "2023"}]))
[:i :y])
(api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}])
(fact "left join on shorts packed into the vector"
(-> (api/left-join (-> (api/dataset [{:iy ["foo" (short 2022)]}]))
(-> (api/dataset [{:iy ["foo" (long 2022)] :s "2022"}
{:iy ["foo" (long 2023)] :s "2023"}]))
:iy)
(api/rows :as-maps)) => [{:iy ["foo", 2022], :right.iy ["foo", 2022], :s "2022"}])
(fact "multiple same rows joins or nils by eraderna"
(-> (api/semi-join (api/dataset [{:k nil :v "\"nil\""}
{:k "bar" :v "\"bar\""}
{:k "baz" :v "\"baz\""}])
(api/dataset [{:k "baz"}])
[:k])
(api/rows :as-maps)) => [{:k "baz" :v "\"baz\""}]
(-> (api/semi-join (api/dataset [{:k nil :v "\"nil\""}
{:k "bar" :v "\"bar\""}
{:k "baz" :v "\"baz\""}])
(api/dataset [{:k "baz"}
{:k "baz"}])
[:k])
(api/rows :as-maps)) => [{:k "baz", :v "\"baz\""}]
(-> (api/semi-join (api/dataset [{:k "bar" :v "\"bar\""}
{:k "baz" :v "\"baz\""}
{:k "baz" :v "\"baz\""}])
(api/dataset [{:k "baz"}])
[:k])
(api/rows :as-maps)) => [{:k "baz", :v "\"baz\""} {:k "baz", :v "\"baz\""}])
tablecloth/test/tablecloth/api/join_separate_test.clj
Vendored
+97
View File
@@ -0,0 +1,97 @@
(ns tablecloth.api.join-separate-test
(:require [tablecloth.api :as api]
[midje.sweet :refer [tabular fact =>]]))
(fact "array-column->columns works"
(-> (api/dataset {:x [(double-array [1 2 3])
(double-array [4 5 6])]
:y [:a :b]})
(api/array-column->columns :x)
(api/rows :as-maps))
=> [{:y :a 0 1.0 1 2.0 2 3.0} {:y :b 0 4.0 1 5.0 2 6.0}]
(-> {"a" [[4 4] [3 2] [2 0]]}
api/dataset
(api/array-column->columns "a" { :prefix "c"})
(api/rows :as-maps)
)
=> [{"c-0" 4, "c-1" 4} {"c-0" 3, "c-1" 2} {"c-0" 2, "c-1" 0}]
)
(fact "array-column->columns works can prefix columns with key-word"
(-> (api/dataset {:x [(double-array [1 2 3])
(double-array [4 5 6])]
:y [:a :b]})
(api/array-column->columns :x {:prefix :col})
(api/rows :as-maps))
=> [{:y :a
:col-0 1.0
:col-1 2.0
:col-2 3.0}
{:y :b
:col-0 4.0
:col-1 5.0
:col-2 6.0}])
(fact "array-column->columns works can prefix columns with string"
(-> (api/dataset {:x [(double-array [1 2 3])
(double-array [4 5 6])]
:y [:a :b]})
(api/array-column->columns :x {:prefix "col"})
(api/rows :as-maps))
=> [{:y :a
"col-0" 1.0
"col-1" 2.0
"col-2" 3.0}
{:y :b
"col-0" 4.0
"col-1" 5.0
"col-2" 6.0}])
(fact "columns->array columns works"
(let [ds (api/dataset {0 [0.0 1 2]
1 [3.0 4 5]
:x [:a :b :c]})
ds-with-array-column
(-> ds
(api/columns->array-column [0 1] :y))]
(:x ds-with-array-column) => [:a :b :c]
(->> ds-with-array-column :y (mapv vec))
=> [[0.0 3.0] [1.0 4.0] [2.0 5.0]]))
(fact "false-is-not-missing"
(-> (api/dataset [{:a "foo" :b true}
{:a "bar" :b false}])
(api/join-columns :join-columns-string [:a :b])
(api/rows)
(flatten))
=> ["foo-true" "bar-false"])
(fact "map-column->columns work"
(->
(api/dataset {:m [{:a 1 :b 2} {:a 3 :b 4}]
"n" [{:a 10 :b 20} {:a 30 :b 40}]})
(api/map-column->columns :m)
(api/rows :as-maps))
=> [{"n" {:a 10, :b 20}, :m-a 1, :m-b 2} {"n" {:a 30, :b 40}, :m-a 3, :m-b 4}]
(->
(api/dataset {:m [{:a 1 :b 2} {:a 3 :b 4}]
"n" [{:a 10 :b 20} {:a 30 :b 40}]})
(api/map-column->columns "n")
(api/rows :as-maps))
=> [{:m {:a 1, :b 2}, "n-a" 10, "n-b" 20} {:m {:a 3, :b 4}, "n-a" 30, "n-b" 40}]
(->
(api/dataset {:m [{:a 1 :b 2 :d 4} {:a 3 :c "hello"}]})
(api/map-column->columns :m)
(api/rows :as-maps))
=> [{:m-a 1, :m-b 2, :m-d 4, :m-c nil}
{:m-a 3, :m-b nil, :m-d nil, :m-c "hello"}])
tablecloth/test/tablecloth/api/missing_test.clj
Vendored
+152
View File
@@ -0,0 +1,152 @@
(ns tablecloth.api.missing-test
(:require [tablecloth.common-test :refer [approx]]
[tablecloth.api :as api]
[tech.v3.datatype.functional :as dfn]
[midje.sweet :refer [tabular fact =>]]))
(fact "empty-missing"
(let [empty-col (api/dataset {:a [nil nil]})]
(-> empty-col api/select-missing api/row-count)
=>
2
(-> empty-col api/drop-missing api/row-count)
=>
0
(-> empty-col (api/replace-missing :a :value 0) api/select-missing api/row-count)
=>
0
(-> empty-col (api/replace-missing :a :value dfn/mean) api/select-missing api/row-count)
=>
2
(-> empty-col api/replace-missing api/select-missing api/row-count)
=>
2))
(def ds (api/dataset {:a [nil nil nil 1.0 2 nil nil nil nil nil 4 nil 11 nil nil]
:b [2 2 2 nil nil nil nil nil nil 13 nil 3 4 5 5]}))
(fact "selection-and-dropping"
(tabular (fact (-> ds ?s api/rows count) => ?cnt)
?cnt ?s
14 api/select-missing
1 api/drop-missing
11 (api/select-missing :a)
4 (api/drop-missing :a)
7 (api/select-missing :b)
8 (api/drop-missing :b)))
(fact "strategy-mid"
(fact (-> ds api/replace-missing api/select-missing api/row-count)
=>
0)
(fact (-> ds (api/replace-missing :a :mid) api/select-missing api/row-count)
=>
7)
(tabular (fact (-> ds ?cmd (api/column ?col) seq)
=> ?xs)
?xs ?cmd ?col
[1.0 1.0 1.0 1.0 2.0 2.0 2.0 2.0 4.0 4.0 4.0 4.0 11.0 11.0 11.0]
(api/replace-missing :a :mid)
:a
[2 2 2 2 2 2 13 13 13 13 13 3 4 5 5]
(api/replace-missing :b :mid)
:b))
(fact "strategy-down"
(tabular (fact (-> ds ?cmd (api/column ?col) seq)
=> ?xs)
?xs ?cmd ?col
[nil nil nil 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0]
(api/replace-missing :a :down)
:a
[2 2 2 2 2 2 2 2 2 13 13 3 4 5 5]
(api/replace-missing :b :down)
:b
[999.0 999.0 999.0 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0]
(api/replace-missing :a :down 999.0)
:a
[4.5 4.5 4.5 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0]
(api/replace-missing :a :down dfn/mean)
:a))
(fact "strategy-up"
(tabular (fact (-> ds ?cmd (api/column ?col) seq)
=> ?xs)
?xs ?cmd ?col
[1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 nil nil]
(api/replace-missing :a :up)
:a
[2 2 2 13 13 13 13 13 13 13 3 3 4 5 5]
(api/replace-missing :b :up)
:b
[1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 999.0 999.0]
(api/replace-missing :a :up 999.0)
:a
[1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 4.5 4.5]
(api/replace-missing :a :up dfn/mean)
:a))
(fact "strategy-lerp"
(tabular (fact (-> ds ?cmd (api/column ?col) (->> (map approx)))
=> ?xs)
?xs ?cmd ?col
(map approx
[1.0 1.0 1.0 1.0 2
(/ 7 3.0) (/ 8 3.0) 3.0 (/ 10 3.0) (/ 11 3.0)
4 7.5 11.0 11.0 11.0])
(api/replace-missing :a :lerp)
:a
(map approx
[2.0 2.0 2.0
3.5714 5.1429 6.7143 8.2857 9.8571 11.4286
13.0 8.0 3.0 4.0 5.0 5.0])
(api/replace-missing :b :lerp)
:b))
(fact "strategy-lerp-time"
(let [dtds (api/dataset {:dt [(java.time.LocalDateTime/of 2020 1 1 1 1 1)
nil nil nil
(java.time.LocalDateTime/of 2020 10 1 1 1 1)]})]
[(java.time.LocalDateTime/of 2020 1 1 1 1 1)
(java.time.LocalDateTime/of 2020 3 9 13 1 1)
(java.time.LocalDateTime/of 2020 5 17 1 1 1)
(java.time.LocalDateTime/of 2020 7 24 13 1 1)
(java.time.LocalDateTime/of 2020 10 1 1 1 1)]
=>
(seq ((api/replace-missing dtds :lerp) :dt))))
(fact "strategy-value"
(tabular (fact (-> ds ?cmd (api/column ?col) seq)
=> ?xs)
?xs ?cmd ?col
[0.0 0.0 0.0 1.0 2.0 0.0 0.0 0.0 0.0 0.0 4.0 0.0 11.0 0.0 0.0]
(api/replace-missing :a :value 0.0)
:a
[4.5 4.5 4.5 1.0 2.0 4.5 4.5 4.5 4.5 4.5 4.0 4.5 11.0 4.5 4.5]
(api/replace-missing :a :value dfn/mean)
:a
[-10.0 -20.0 -10.0 1.0 2.0
-20.0 -10.0 -20.0 -10.0 -20.0 4.0 -10.0 11.0 -20.0 -10.0]
(api/replace-missing :a :value [-10.0 -20.0])
:a
[nil 100.0 nil 1.0 2.0
nil nil nil nil
-100.0 4.0 nil 11.0 nil nil]
(api/replace-missing :a :value {1 100.0 9 -100.0})
:a))
;; otfrom case: https://clojurians.zulipchat.com/#narrow/stream/151924-data-science/topic/Simple.20Tablecloth.20beginnings/near/249217023
(fact "replace missing values in grouped dataset"
(fact (-> (api/dataset [{:calendar-year 2022 :age 49 :location "Barry Buddon" :hours-cycled 1.0}
{:calendar-year 2024 :age 49 :location "Barry Buddon" :hours-cycled 2.4}
{:calendar-year 2022 :age 49 :location "Tentsmuir Woods" :hours-cycled 3.2}
{:calendar-year 2024 :age 49 :location "Tentsmuir Woods" :hours-cycled 1.4}])
(api/group-by [:age :location])
(api/fill-range-replace :calendar-year 1 nil nil)
(api/replace-missing :location :down)
(api/replace-missing :age :down)
(api/replace-missing :hours-cycled :value 0.0)
(api/ungroup)
(api/rows :as-maps)) => [{:calendar-year 2022.0, :age 49, :location "Barry Buddon", :hours-cycled 1.0} {:calendar-year 2023.0, :age 49, :location "Barry Buddon", :hours-cycled 0.0} {:calendar-year 2024.0, :age 49, :location "Barry Buddon", :hours-cycled 2.4} {:calendar-year 2022.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 3.2} {:calendar-year 2023.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 0.0} {:calendar-year 2024.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 1.4}]))
tablecloth/test/tablecloth/api/operators_test.clj
Vendored
+177
View File
@@ -0,0 +1,177 @@
(ns tablecloth.api.operators-test
(:refer-clojure :exclude [* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip
bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor
even? identity infinite? max min neg? not odd? odd? or pos? quot rem
unsigned-bit-shift-right zero?])
(:require [tablecloth.api :refer [dataset]]
[tech.v3.datatype :refer [elemwise-datatype]]
[midje.sweet :refer [fact facts =>]])
(:use [tablecloth.api.operators]))
(facts
"about ops that return the op result"
(facts
"about ops that take a maximum of one column and return a scalar"
(let [ds (dataset {:label [:a :b :a :b]
:value [1 2 3 4]})]
(let [ops [kurtosis
magnitude
magnitude-squared
mean
mean-fast
median
quartile-1
quartile-3
reduce-*
reduce-+
reduce-max
reduce-min
skew
sum
sum-fast
variance]]
(doseq [op ops]
(fact "ops can aggregate a regular dataset"
(let [result (op ds [:value])]
result => tablecloth.api/dataset?
(contains? result "summary") => true))
(fact "ops can aggregate a grouped ds"
(let [result (-> ds
(tablecloth.api/group-by :label)
(op [:value]))]
result => tablecloth.api/dataset?
(contains? result "summary") => true))))))
(facts
"about ops that take a maximum of two columns and return a scalar"
(let [ds (dataset {:label [:a :b :a :b]
:values1 [1 2 3 4]
:values2 [5 6 7 8]})]
(let [ops [distance
distance-squared
dot-product]]
(doseq [op ops]
(fact "ops can aggregate a regular dataset"
(let [result (op ds [:values1 :values2])]
result => tablecloth.api/dataset?
(contains? result "summary")))
(fact "ops can aggregate a grouped dataset"
(let [result (-> ds
(tablecloth.api/group-by :label)
(op [:values1 :values2]))]
result => tablecloth.api/dataset?
(contains? result "summary"))))))))
(facts
"about ops that return a dataset with a new column"
(facts
"about ops that take a maximum of one column"
(let [ds (dataset {:a [1 2 3]})]
(let [result (shift ds :b [:a] 1)]
(:b result) => [1 1 2])
(let [ops [cummax
cummin
cumprod
cumsum
abs
acos
asin
atan
bit-not
cbrt
ceil
cos
cosh
even?
exp
expm1
finite?
floor
get-significand
identity
infinite?
log
log10
log1p
logistic
mathematical-integer?
nan?
neg?
next-down
next-up
normalize
not
odd?
pos?
rint
round
signum
sin
sinh
sq
sqrt
tan
tanh
to-degrees
to-radians
ulp
zero?]]
(doseq [op ops]
(let [result (op ds :b [:a])]
(contains? result :b) => true)))))
(facts
"about ops that take a maximum of two columns"
(let [ops [and eq not-eq or]
ds (dataset {:a [1 2 3]
:b [4 5 6]})]
(doseq [op ops]
(let [result (op ds :c [:a :b :c])]
(contains? result :c) => true))))
(facts
"about ops that take a maximum of three columns"
(let [ops [< > <= >=]
ds (dataset {:a [1 2 3]
:b [4 5 6]
:c [7 8 9]})]
(doseq [op ops]
(let [result (op ds :d [:a :b :c])]
(contains? result :d) => true))))
(facts
"about ops that can take an unlimited number of columns"
(let [ops [/
-
+
*
atan2
;; equals // this function doesn't seem to handle more than two cols
hypot
bit-and
bit-and-not
bit-clear
bit-flip
bit-or
bit-set
bit-shift-right
bit-shift-left
bit-xor
ieee-remainder
max
min
pow
quot
rem
unsigned-bit-shift-right]
ds (dataset {:a [1 2 3]
:b [4 5 6]
:c [7 8 9]
:d [10 11 12]})]
(doseq [op ops]
(let [result (op ds :e [:a :b :c :d])]
(contains? result :e) => true)))))
tablecloth/test/tablecloth/api/reshape_test.clj
Vendored
+358
View File
@@ -0,0 +1,358 @@
(ns tablecloth.api.reshape-test
(:require [tablecloth.api :as api]
[tech.v3.datatype.functional :as dfn]
[clojure.string :refer [starts-with?]]
[midje.sweet :refer [tabular fact =>]]))
;; tidyr tests
(fact "pivot-longer-basic"
(let [ds (-> {:x [1 2] :y [3 4]}
(api/dataset)
(api/pivot->longer))]
(fact (api/column-names ds)
=> '(:$column :$value))
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
[:x :x :y :y] :$column
[1 2 3 4] :$value)))
(fact "pivot-longer-column-names"
(let [ds (-> {:x [1 2] :y [3 4]}
(api/dataset)
(api/pivot->longer :all {:target-columns :colname
:value-column-name :v}))]
(fact (api/column-names ds)
=> '(:colname :v))
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
[:x :x :y :y] :colname
[1 2 3 4] :v)))
(fact "pivot-longer-datatypes"
(let [ds (-> {:x [1 2] :y [3 4]}
(api/dataset)
(api/pivot->longer :all {:datatypes {:$column [:object (comp symbol name)]
:$value [:float32 double]}}))]
(fact (api/column-names ds)
=> '(:$column :$value))
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
'(x x y y) :$column
[1.0 2.0 3.0 4.0] :$value)))
(fact "pivot-longer-keep-column"
(let [ds (-> {:x [1 2] :y 2 :z [1 2]}
(api/dataset)
(api/pivot->longer [:y :z]))]
(fact (api/column-names ds)
=> '(:x :$column :$value))
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
[1 2 1 2] :x
[:y :y :z :z] :$column
[2 2 1 2] :$value)))
(fact "pivot-longer-missing"
(let [ds (-> {:x [1 nil] :y [nil 2]}
(api/dataset))
ds-drop (api/pivot->longer ds)
ds-keep (api/pivot->longer ds :all {:drop-missing? false})]
(tabular (fact (seq (?d ?n))
=> ?v)
?v ?d ?n
[:x :y] ds-drop :$column
[1 2] ds-drop :$value
[:x :x :y :y] ds-keep :$column
[1 nil nil 2] ds-keep :$value)))
(fact "pivot-longer-separate-names-with-missing-combinations"
(let [ds (-> {:id ["A" "B"]
"x_1" [1 3]
"x_2" [2 4]
"y_2" ["a" "b"]}
(api/dataset))
ds-drop (api/pivot->longer ds string? {:target-columns [nil "n"]
:splitter "_"})
ds-keep (api/pivot->longer ds string? {:target-columns [nil "n"]
:splitter "_"
:drop-missing? false})]
(tabular (fact (seq (?d ?n))
=> ?v)
?v ?d ?n
["A" "B" "A" "B"] ds-keep :id
[1 1 2 2] ds-keep "n"
[1 3 2 4] ds-keep "x"
[nil nil "a" "b"] ds-keep "y"
["A" "B"] ds-drop :id
[2 2] ds-drop "n"
[2 4] ds-drop "x"
["a" "b"] ds-drop "y")))
(fact "pivot-longer-separate-names"
(let [ds (-> {:id ["A" "B"]
"z_1" [1 7]
"y_1" [2 8]
"x_1" [3 9]
"z_2" [4 10]
"y_2" [5 11]
"x_2" [6 12]}
(api/dataset)
(api/pivot->longer string? {:target-columns [nil :n]
:splitter "_"}))]
(fact (api/column-names ds)
=> [:id :n "z" "y" "x"])
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
["A" "B" "A" "B"] :id
[1 1 2 2] :n
[1 7 4 10] "z"
[2 8 5 11] "y"
[3 9 6 12] "x")))
(fact "pivot-longer-separate-names-regex"
(let [ds (-> {:id ["A" "B"]
"z_1" [1 7]
"y_1" [2 8]
"x_1" [3 9]
"z_2" [4 10]
"y_2" [5 11]
"x_2" [6 12]}
(api/dataset)
(api/pivot->longer string? {:target-columns [nil :n]
:splitter #"(.)_(.)"}))]
(fact (api/column-names ds)
=> [:id :n "z" "y" "x"])
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
["A" "B" "A" "B"] :id
[1 1 2 2] :n
[1 7 4 10] "z"
[2 8 5 11] "y"
[3 9 6 12] "x")))
(fact "pivot-longer-separate-names_vector"
(let [ds (-> {:id ["A" "B"]
[:x 1] [1 3]
[:x 2] [2 4]
[:y 2] ["a" "b"]}
(api/dataset)
(api/pivot->longer vector? {:target-columns [nil :n]
:splitter identity}))]
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
["A" "B"] :id
[2 2] :n
[2 4] :x
["a" "b"] :y)))
(fact "pivot-longer-separate-names-custom"
(let [ds (-> {:id ["A" "B"]
{:a :x :b 1} [1 3]
{:a :x :b 2} [2 4]
{:a :y :b 2} ["a" "b"]}
(api/dataset)
(api/pivot->longer map? {:target-columns [nil :n]
:splitter (juxt :a :b)}))]
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
["A" "B"] :id
[2 2] :n
[2 4] :x
["a" "b"] :y)))
;;
(fact "pivot-wider-basic"
(let [ds (-> {:key ["x" "y" "z"]
:val [1 2 3]}
(api/dataset)
(api/pivot->wider :key :val))]
(tabular (fact (first (ds ?n))
=> ?v)
?v ?n
1 "x"
2 "y"
3 "z")))
(fact "pivot-wider-not-pivoted-cols"
(let [ds (-> {:a 1
:key ["x" "y"]
:val [1 2]}
(api/dataset)
(api/pivot->wider :key :val))]
(fact (api/column-names ds)
=> [:a "x" "y"])
(tabular (fact (first (ds ?n))
=> ?v)
?v ?n
1 :a
1 "x"
2 "y")))
(fact "pivot-wider-missings"
(let [ds (-> {:a [1 2]
:key ["x" "y"]
:val [1 2]}
(api/dataset)
(api/pivot->wider :key :val {:drop-missing? false}))]
(fact (api/column-names ds)
=> [:a "x" "y"])
(tabular (fact (seq (ds ?n))
=> ?v)
?v ?n
[2 1] :a
[nil 1] "x"
[2 nil] "y")))
(fact "pivot-wider-concat-columns"
(let [ds (-> {:x ["X" "Y"]
:y [1 2]
:a [1 2]
:b [1 2]}
(api/dataset)
(api/pivot->wider [:x :y] [:a :b] {:concat-columns-with ""
:concat-value-with "_"}))]
(fact (api/column-names ds)
=> ["X1_a" "X1_b" "Y2_a" "Y2_b" ])
(fact (api/rows ds)
=> [[1 1 2 2]])))
(fact "pivot-wider-multiple"
(let [ds (-> {:a [1 1 2]
:key ["x" "x" "x"]
:val [1 2 3]}
(api/dataset)
(api/pivot->wider :key :val {:fold-fn vec}))]
(fact (seq (ds "x"))
=> [[1 2] [3]])))
(fact "pivot-wider-multiple-summarize"
(let [ds (-> {:a [1 1 2]
:key ["x" "x" "x"]
:val [1 10 100]}
(api/dataset)
(api/pivot->wider :key :val {:fold-fn dfn/sum}))]
(fact (seq (ds "x"))
=> [11.0 100.0])))
(fact "pivot-wider-multiple-values"
(let [ds (-> {:row 1
:var [:x :y]
:a [1 2]
:b [3 4]}
(api/dataset)
(api/pivot->wider :var [:a :b]))]
(fact (api/rows ds :as-maps)
=> [{:row 1, "x-b" 3, "y-a" 2, "x-a" 1, "y-b" 4}])))
(fact "pivot-wider-multiple-values-no-row"
(let [ds (-> {:var [:x :y]
:a [1 2]
:b [3 4]}
(api/dataset)
(api/pivot->wider :var [:a :b]))]
(fact (api/rows ds :as-maps)
=> [{"x-b" 3, "y-a" 2, "x-a" 1, "y-b" 4}])))
;; construction case
(defonce construction (api/dataset "data/construction.csv"))
(def construction-unit-map {"1 unit" "1"
"2 to 4 units" "2-4"
"5 units or more" "5+"})
(fact "pivot-revert"
(let [ds (-> construction
(api/pivot->longer #"^[125NWS].*|Midwest" {:target-columns [:units :region]
:splitter (fn [col-name]
(if (re-matches #"^[125].*" col-name)
[(construction-unit-map col-name) nil]
[nil col-name]))
:value-column-name :n
:drop-missing? false
:coerce-to-number? false})
(api/pivot->wider [:units :region] :n {:drop-missing? false})
(api/rename-columns (zipmap (vals construction-unit-map)
(keys construction-unit-map))))]
(fact (api/row-count construction)
=> (api/row-count ds))
(fact (api/column-count construction)
=> (api/column-count ds))
(fact (set (api/column-names construction))
=> (set (api/column-names ds)))
(tabular (fact (seq (construction ?c))
=> (seq (ds ?c)))
?c
"Year" "Month" "1 unit" "2 to 4 units" "5 units or more" "Northeast" "Midwest" "South" "West")))
(fact "pivot-sizes"
(tabular (fact (api/shape (-> ?d (api/dataset) ?f))
=> ?s)
?s ?d ?f
[180 3] "data/relig_income.csv"
(api/pivot->longer (complement #{"religion"}))
[5307 5] (-> (api/dataset "data/billboard.csv.gz")
(api/drop-columns :type/boolean))
(api/pivot->longer #(starts-with? % "wk") {:target-columns :week
:value-column-name :rank})
[76046 8] "data/who.csv.gz"
(api/pivot->longer #(starts-with? % "new") {:target-columns [:diagnosis :gender :age]
:splitter #"new_?(.*)_(.)(.*)"
:value-column-name :count})
[405440 8] "data/who.csv.gz"
(api/pivot->longer #(starts-with? % "new") {:target-columns [:diagnosis :gender :age]
:splitter #"new_?(.*)_(.)(.*)"
:value-column-name :count
:drop-missing? false})
[9 4] "data/family.csv"
(api/pivot->longer (complement #{"family"}) {:target-columns [nil :child]
:splitter "_"
:datatypes {"gender" :int16}})
[44 3] "data/anscombe.csv"
(api/pivot->longer :all {:splitter #"(.)(.)"
:target-columns [nil :set]})
[8 6] (api/dataset {:x [1 2 3 4]
:a [1 1 0 0]
:b [0 1 1 1]
:y1 (repeatedly 4 rand)
:y2 (repeatedly 4 rand)
:z1 [3 3 3 3]
:z2 [-2 -2 -2 -2]})
(api/pivot->longer [:y1 :y2 :z1 :z2] {:target-columns [nil :times]
:splitter #":(.)(.)"})
[19 12] "data/fish_encounters.csv"
(api/pivot->wider "station" "seen" {:drop-missing? false})
[6 3] "data/warpbreaks.csv"
(-> (api/group-by ["wool" "tension"])
(api/aggregate {:n api/row-count}))
[3 3] "data/warpbreaks.csv"
(-> (api/reorder-columns ["wool" "tension" "breaks"])
(api/pivot->wider "wool" "breaks" {:fold-fn vec}))
[15 4] "data/production.csv"
(api/pivot->wider ["product" "country"] "production")
[52 6] "data/us_rent_income.csv"
(api/pivot->wider "variable" ["estimate" "moe"] {:drop-missing? false})
[3 4] "data/contacts.csv"
(api/pivot->wider "field" "value" {:drop-missing? false})))
tablecloth/test/tablecloth/api/utils_test.clj
Vendored
+140
View File
@@ -0,0 +1,140 @@
(ns tablecloth.api.utils-test
(:require [tablecloth.api :as api]
[tablecloth.api.utils :as sut]
[tech.v3.dataset :as ds]
[tech.v3.datatype :as dtype]
[clojure.string :as str]
[midje.sweet :refer [tabular fact =>]]))
(def ds (api/dataset "data/who.csv.gz"))
(def dsg (api/group-by ds "country"))
(def ds-sn (api/dataset {[1 2 3] [1 2 3]
false ["false" "false" "false"]
true ["true" "true" "true"]
{:a 1 :b 2} [9 9 9]}))
(fact "iterable-sequence"
(tabular (fact (sut/iterable-sequence? ?x)
=> true)
[?x]
[] '() #{}
(ds "country")
(java.util.ArrayList.)
(dtype/const-reader 3 5))
(tabular (fact (sut/iterable-sequence? ?x)
=> false)
[?x]
nil {}))
(fact "->str"
(tabular (fact (sut/->str ?y)
=> ?x)
[?x ?y]
"a" "a"
"b" :a/b
"c" 'a/c
"b" :b
"c" 'c))
(fact "column-names-regular"
(fact (api/column-names ds)
=> (ds/column-names ds))
(fact (api/column-names ds :all)
=> (ds/column-names ds))
(fact (nil? (api/column-names ds nil))
=> true)
(tabular (fact (api/column-names ds ?y)
=> ?x)
[?x ?y]
["country" "iso2" "iso3"] :type/string
["country" "iso2" "iso3"] :!type/numerical
["year"] "year"
["year"] ["year"]
["iso2" "year"] #{"year" "iso2"}
["new_sp_m014" "country"] ["new_sp_m014" "country"]
["iso2" "iso3"] #(str/starts-with? % "iso")
["country"] {"country" nil} ;; only keys are used
["iso2" "iso3"] #"^i.*")
(fact (every? #(str/starts-with? % "new") (api/column-names ds (complement #{"country" "iso2" "iso3" "year"})))
=> true)
(tabular (fact (api/column-names ds ?y ?f)
=> ?x)
[?x ?y ?f]
["iso2" "iso3"] #{"iso2" "iso3"} :name
["country" "iso2" "iso3"] #{:string} :datatype
["new_sp_m014"] (fn [{:keys [name datatype]}]
(and (= datatype :int16)
(str/starts-with? name "new_sp_m"))) :all)
(tabular (fact (api/column-names ds-sn ?y)
=> ?x)
[?x ?y]
[false] false?
[true] true?
[false] #{false}
[true] #{true}
[[1 2 3]] vector?
[{:a 1 :b 2}] map?
[[1 2 3] {:a 1 :b 2}] seqable?
[false true] boolean?
[[1 2 3] {:a 1 :b 2}] :type/numerical
[false true] :!type/numerical))
(fact "column-names-grouped"
(fact (api/column-names dsg)
=> (ds/column-names ds))
(fact (api/column-names dsg :all)
=> (ds/column-names ds))
(fact (nil? (api/column-names dsg nil))
=> true)
(tabular (fact (api/column-names dsg ?y)
=> ?x)
[?x ?y]
["country" "iso2" "iso3"] :type/string
["year"] "year"
["year"] ["year"]
["iso2" "year"] #{"year" "iso2"}
["new_sp_m014" "country"] ["new_sp_m014" "country"]
["iso2" "iso3"] #(str/starts-with? % "iso")
["country"] {"country" nil} ;; only keys are used
["iso2" "iso3"] #"^i.*")
(fact (every? #(str/starts-with? % "new") (api/column-names dsg (complement #{"country" "iso2" "iso3" "year"})))
=> true)
(tabular (fact (api/column-names dsg ?y ?f)
=> ?x)
[?x ?y ?f]
["iso2" "iso3"] #{"iso2" "iso3"} :name
["country" "iso2" "iso3"] #{:string} :datatype
["new_sp_m014"] (fn [{:keys [name datatype]}]
(and (= datatype :int16)
(str/starts-with? name "new_sp_m"))) :all))
(fact "rank-tests"
(let [x2 [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]]
(tabular (fact (sut/rank ?y)
=> ?x)
[?x ?y]
'(1.0 0.0 2.0 3.0 4.0) [3 1 4 15 92]
'(1.0 0.0 2.0 3.0 4.0) '(1.0 0.0 2.0 3.0 4.0)
'(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0) x2
'(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0) '(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0)
;; as in data.table, `frankv(?x, na.last=FALSE)`
'(5.0 2.5 5.0 0.5 2.5 0.5 5.0) [4 1 4 nil 1 nil 4])
(tabular (fact (sut/rank ?y ?ties)
=> ?x)
[?x ?y ?ties]
[3 0 5 1 6 10 2 9 7 4 8] x2 :first
[4 1 5 0 8 10 2 9 7 3 6] x2 :last
'(4 1 5 1 8 10 2 9 8 4 8) x2 :max
'(3 0 5 0 6 10 2 9 6 3 6) x2 :min
'(2 0 3 0 4 6 1 5 4 2 4) x2 :dense)
(fact (sut/rank x2 :average true)
=> '(6.5 9.5 5.0 9.5 3.0 0.0 8.0 1.0 3.0 6.5 3.0))
(fact (sut/rank x2 :dense true)
=> '(4 6 3 6 2 0 5 1 2 4 2))))
(fact "->general-types describes the set of general types for a concrete datatype"
(sut/->general-types :int32) => #{:integer :numerical}
(sut/->general-types :float32) => #{:float :numerical}
(sut/->general-types :string) => #{:textual})
tablecloth/test/tablecloth/column/api/column_test.clj
Vendored
+89
View File
@@ -0,0 +1,89 @@
(ns tablecloth.column.api.column-test
(:require [tablecloth.column.api.column :refer [column zeros ones typeof?
typeof slice sort-column]]
[tech.v3.dataset.column :refer [is-column?]]
[midje.sweet :refer [fact facts =>]]))
(fact "`column` returns a column"
(is-column? (column)) => true)
(fact "`column` provides a few ways to generate a column`"
(column) => []
(column [1 2 3]) => [1 2 3]
(column (list 1 2 3)) =>[1 2 3]
(column (range 3)) => [0 1 2])
(fact "`column` identifies the type of the elements automatically"
(-> [1 2 3]
(column)
(tech.v3.datatype/elemwise-datatype)) => :int64
(-> [true false true]
(column)
(tech.v3.datatype/elemwise-datatype)) => :boolean
(-> [1 true false]
(column)
(tech.v3.datatype/elemwise-datatype)) => :object)
(fact "`typeof` returns the concrete type of the elements"
(typeof (column [1 2 3])) => :int64
(typeof (column ["a" "b" "c"])) => :string
(typeof (column [true false])) => :boolean)
(fact "`typeof?` can check the concerete type of column elements"
(typeof? (column [1 2 3]) :int64) => true
(typeof? (column [1 2 3]) :int32) => false
(typeof? (column ["a" "b" "c"]) :string) => true)
(fact "`typeof?` can check the general type of column elements"
(typeof? (column [1 2 3]) :integer) => true
(typeof? (column [1 2 3]) :textual) => false
(typeof? (column [1.0 2.0 3.0]) :numerical) => true
(typeof? (column [1.0 2.0 3.0]) :logical) => false
(typeof? (column ["a" "b" "c"]) :textual) => true
(typeof? (column ["a" "b" "c"]) :numerical) => false
(typeof? (column [true false true]) :logical) => true)
(fact "`zeros` returns a column filled with zeros"
(zeros 3) => [0 0 0])
(fact "`ones` returns a column filled with ones"
(ones 3) => [1 1 1])
(facts "about `slice`"
(let [c (column [1 2 3 4 5])]
(fact "it return a subset of a column inclusively"
(slice c 0 0) => [1]
(slice c 0 4) => [1 2 3 4 5])
(fact "it supports negative indexing inclusively"
(slice c 0 -1)
(slice c -1 -1) => [5]
(slice c -3 -1) => [3 4 5])
(fact "it supports 0 within negative indexing"
(slice c 0 -2) => [1 2 3 4])
(fact "it supports stepped slicing"
(slice c 0 4 2) => [1 3 5])
(fact "it supports using nil to indicate slice from start or end"
(slice c 2) => [3 4 5]
(slice c -2) => [4 5]
(slice c nil 2) => [1 2 3]
(slice c nil -2) => [1 2 3 4])
(fact "it supports special keywords for selecting from start or end"
(slice c :start 2) => [1 2 3]
(slice c 1 :end) => [2 3 4 5]
(slice c -4 :end) => [2 3 4 5]
(slice c :start -3) => [1 2 3])))
(facts "about `sort-column`"
(let [c-ints (column [3 100 9 0 -10 43])
c-strings (column ["a" "z" "baz" "fo" "bar" "foo"])]
(fact "it returns a column"
(sort-column c-ints) => is-column?)
(fact "it sorts in ascending order by default"
(sort-column c-ints) => [-10 0 3 9 43 100]
(sort-column c-strings) => ["a" "bar" "baz" "fo" "foo" "z"])
(fact "it accepts a comparator-fn"
(sort-column c-strings
#(> (count %1) (count %2))) => ["baz" "bar" "foo" "fo" "z" "a"])
(fact "it moves missing values to the end"
(sort-column (column [nil 100 nil 3 -10])) => [-10 3 100 nil nil])))
tablecloth/test/tablecloth/column/api/missing_test.clj
Vendored
+13
View File
@@ -0,0 +1,13 @@
(ns tablecloth.column.api.missing-test
(:require [tablecloth.column.api.missing :refer [count-missing replace-missing drop-missing]]
[tablecloth.column.api.column :refer [column]]
[midje.sweet :refer [facts =>]]))
(facts "about `count-missing`"
(count-missing (column [1 2 3])) => 0)
(facts "about `replace-missing`"
(replace-missing (column [1 nil 3])) => [1 1 3])
(facts "about `drop-missing`"
(drop-missing (column [1 nil 3])) => [1 3])
tablecloth/test/tablecloth/column/api/operators_test.clj
Vendored
+219
View File
@@ -0,0 +1,219 @@
(ns tablecloth.column.api.operators-test
(:refer-clojure :exclude [* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip
bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor
even? identity infinite? max min neg? not odd? odd? or pos? quot rem
unsigned-bit-shift-right zero?])
(:require [midje.sweet :refer [fact facts =>]]
[clojure.test :refer [deftest is]]
[tablecloth.column.api :refer [column column? typeof]])
(:use [tablecloth.column.api.operators]))
(defn sample-column [n]
(column (repeatedly n #(rand-int 100))))
(defn scalar? [item]
(= (tech.v3.datatype.argtypes/arg-type item)
:scalar))
(facts
"about 'shift"
(let [a (column [1 2 3 4 5])]
(shift a 2) => [1 1 1 2 3]))
(facts
"about 'descriptive-statistics"
(let [a (sample-column 5)]
;; sanity check that we got the hash with desired data
(descriptive-statistics a) => #(contains? % :standard-deviation)))
(facts
"about 'quartiles"
(let [a (sample-column 100)]
(quartiles a) => column?
;; sanity check quartiles should return a coumn of 5 values
(count (quartiles a)) => 5))
(facts
"about 'fill-range"
(let [result (fill-range [1 5] 1)]
(contains? result :result) => true
(contains? result :missing) => true))
(facts
"about ops that take a single column and return a column"
(let [ops [abs
acos
asin
atan
bit-not
cbrt
ceil
cos
cosh
cumprod
cumsum
cummax
cummin
exp
expm1
floor
get-significand
identity
log
log10
log1p
logistic
next-down
next-up
normalize
rint
signum
sin
sinh
sq
sqrt
tan
tanh
to-degrees
to-radians
ulp
]
a (sample-column 5)]
(doseq [op ops]
(op a) => column?)))
(facts
"about ops that take a single column and return a scalar"
(let [ops [magnitude
reduce-max
reduce-min
reduce-*
reduce-+
mean-fast
sum-fast
magnitude-squared]
a (sample-column 5)]
(doseq [op ops]
(op a) => scalar?)))
(facts
"about ops that take one or more columns or scalars and return either a scalar or a column"
(let [ops [/ - +]
a (sample-column 5)
b (sample-column 5)
c (sample-column 5)
d (sample-column 5)]
(doseq [op ops]
(op a) => column?
(op a b) => column?
(op a b c) => column?
(op a b c d) => column?
(op 1) => scalar?
(op 1 2) => scalar?
(op 1 2 3) => scalar?)))
(facts
"about comparison ops that take two or more columns and return a boolean"
(let [ops [> >= < <=]
a (sample-column 5)
b (sample-column 5)
c (sample-column 5)]
(doseq [op ops]
(op a b) => column?
(op a b c) => column?
(op 1 2) => boolean?)))
(facts
"about comparison ops that take two columns and return a boolean"
(let [ops [equals]
a (sample-column 5)
b (sample-column 5)]
(doseq [op ops]
(op a b) => boolean?)))
(facts
"about comparison ops that take two columns or scalars and return a boolean or column of booleans"
(let [ops [or and eq not-eq]
a (sample-column 5)
b (sample-column 5)]
(doseq [op ops]
(op a b) => column?
(typeof (op a b)) => :boolean
(op 1 2) => boolean?)))
(facts
"about ops that take a single column or scalar and return a scalar"
(let [ops [kurtosis
sum
mean
skew
variance
standard-deviation
quartile-3
quartile-1
median]
a (sample-column 5)]
(doseq [op ops]
(op a) => scalar?)))
(facts
"about ops that take two or more scalars or columns and return a column or scalar"
(let [ops [*
bit-and
bit-and-not
bit-clear
bit-flip
bit-or
bit-set
bit-shift-right
bit-shift-left
;; bit-test
bit-xor
hypot
ieee-remainder
max
min
pow
quot
rem
unsigned-bit-shift-right
]
a (sample-column 5)
b (sample-column 5)
c (sample-column 5)]
(doseq [op ops]
(op a b) => column?
(op a b c) => column?
(op 5 5)
(op 5 5 5))))
(facts
"about ops that take left-hand / right-hand columns and return a scalar"
(let [ops [distance
dot-product
distance-squared
kendalls-correlation
pearsons-correlation
spearmans-correlation]
a (sample-column 5)
b (sample-column 5)]
(doseq [op ops]
(op a b) => scalar?)))
(facts
"about ops that take a single column or scalar and return boolean or column of booleans"
(let [ops [finite?
pos?
neg?
mathematical-integer?
nan?
even?
zero?
not
infinite?
odd?]
a (sample-column 5)]
(doseq [op ops]
(op a) => column?
(typeof (op a)) => :boolean
(op 1) => boolean?)))
tablecloth/test/tablecloth/common_test.clj
Vendored
+16
View File
@@ -0,0 +1,16 @@
(ns tablecloth.common-test
(:require [tablecloth.api :as api]))
(def DS (api/dataset {:V1 (take 9 (cycle [1 2]))
:V2 (range 1 10)
:V3 (take 9 (cycle [0.5 1.0 1.5]))
:V4 (take 9 (cycle ["A" "B" "C"]))}
{:dataset-name "DS"}))
(defn approx
^double [^double v]
(-> (Double/toString v)
(BigDecimal.)
(.setScale 4 BigDecimal/ROUND_HALF_UP)
(.doubleValue)))