init research

tech.ml.dataset/test/tech/v3/libs/arrow_test.clj

@@ -0,0 +1,354 @@
+(ns tech.v3.libs.arrow-test
+  (:require [tech.v3.libs.arrow :as arrow]
+            [tech.v3.dataset :as ds]
+            [tech.v3.dataset.column :as ds-col]
+            [tech.v3.dataset.impl.sparse-column :as sparse-col]
+            [tech.v3.datatype.functional :as dfn]
+            [tech.v3.datatype :as dtype]
+            [tech.v3.libs.parquet]
+            [tech.v3.datatype.datetime :as dtype-dt]
+            [tech.v3.resource :as resource]
+            [clojure.test :refer [deftest is]])
+  (:import [java.time LocalTime]
+           [tech.v3.dataset Text]
+           [java.util Map]
+           [java.io ByteArrayOutputStream ByteArrayInputStream]))
+
+
+(tech.v3.dataset.utils/set-slf4j-log-level :info)
+
+
+(defn supported-datatype-ds
+  ([n]
+   (-> (ds/->dataset {:boolean [true false true true false false true false false true]
+                      :bytes (byte-array (range n))
+                      :ubytes (dtype/make-container :uint8 (dfn/rem (range n) 256))
+                      :shorts (short-array (range n))
+                      :ushorts (dtype/make-container :uint16 (range n))
+                      :ints (int-array (range n))
+                      :uints (dtype/make-container :uint32 (range n))
+                      :longs (long-array (range n))
+                      :floats (float-array (range n))
+                      :doubles (double-array (range n))
+                      :strings (map str (range n))
+                      :text (map (comp #(Text. %) str) (range n))
+                      :instants (repeatedly n dtype-dt/instant)
+                      :bigdec (repeatedly n #(BigDecimal/valueOf (+ 100 (rand-int 1700)) 2))
+                      ;; :bigint (let [rng (java.util.Random.)]
+                      ;;           (repeatedly n #(BigInteger. 256 rng )))
+                      ;;external formats often don't support dash-case
+                      :local_dates (repeatedly n dtype-dt/local-date)
+                      :local_times (repeatedly n dtype-dt/local-time)
+                      :uuids (repeatedly n #(java.util.UUID/randomUUID))})
+       (vary-meta assoc :name :testtable)))
+  ([]
+   (supported-datatype-ds 10)))
+
+
+
+(comment
+  (arrow/dataset->stream! (supported-datatype-ds 1000) "test/data/alldtypes.arrow-ipc-lz4"
+                          {:compression :lz4})
+
+  (arrow/dataset->stream! (supported-datatype-ds 1000) "test/data/alldtypes.arrow-ipc-zstd"
+                          {:compression :zstd})
+
+
+  (let [sds (supported-datatype-ds 1000)]
+    (arrow/dataset-seq->stream! "test/data/alldtypes.arrow-file-zstd"
+                                {:compression :zstd
+                                 :format :file
+                                 :strings-as-text? true}
+                                [(ds/select-rows sds (range 500))
+                                 ;;test when you have to add more string dictionary values
+                                 (ds/select-rows sds (range 500 1000))]))
+
+  (def ignored (arrow/stream->dataset-seq "test/data/alldtypes.arrow-file-zstd"))
+
+  (def ignored (arrow/stream->dataset "test/data/alldtypes.arrow-ipc-zstd"))
+
+  )
+
+
+(deftest base-datatype-test
+  (try
+    (resource/stack-resource-context
+     (let [ds (supported-datatype-ds)
+           _ (arrow/dataset->stream! ds "alldtypes.arrow")
+           mmap-ds (arrow/stream->dataset "alldtypes.arrow" {:open-type :mmap
+                                                             :key-fn keyword})
+           copy-ds (arrow/stream->dataset "alldtypes.arrow" {:key-fn keyword})]
+       (doseq [col (vals ds)]
+         (let [cname ((meta col) :name)
+               dt (dtype/elemwise-datatype col)
+               inp-col (mmap-ds cname)
+               cp-col (copy-ds cname)]
+           (is (= dt (dtype/elemwise-datatype inp-col)) (str "inplace failure " cname))
+           (is (= dt (dtype/elemwise-datatype cp-col)) (str "copy failure " cname))
+
+           (is (= (vec col) (vec inp-col)) (str "inplace failure " cname))
+           (is (= (vec col) (vec cp-col)) (str "copy failure " cname))))))
+    (finally
+      (.delete (java.io.File. "alldtypes.arrow")))))
+
+
+(deftest base-sparse-datatype-test
+  (try
+    (resource/stack-resource-context
+      (let [ds (sparse-col/->sparse-ds (supported-datatype-ds) 0.0)
+            _ (arrow/dataset->stream! ds "alldtypes-sparse.arrow")
+            mmap-ds (arrow/stream->dataset "alldtypes-sparse.arrow" {:open-type :mmap
+                                                                     :key-fn keyword})
+            copy-ds (arrow/stream->dataset "alldtypes-sparse.arrow" {:key-fn keyword})]
+        (is (every? sparse-col/is-sparse? (.values ^Map ds)))
+        (is (every? sparse-col/is-sparse? (.values ^Map mmap-ds)))
+        (is (every? sparse-col/is-sparse? (.values ^Map copy-ds)))
+        (doseq [col (vals ds)]
+          (let [cname ((meta col) :name)
+                dt (dtype/elemwise-datatype col)
+                inp-col (mmap-ds cname)
+                cp-col (copy-ds cname)]
+            (is (= dt (dtype/elemwise-datatype inp-col)) (str "inplace failure " cname))
+            (is (= dt (dtype/elemwise-datatype cp-col)) (str "copy failure " cname))
+
+            (is (= (vec col) (vec inp-col)) (str "inplace failure " cname))
+            (is (= (vec col) (vec cp-col)) (str "copy failure " cname))))))
+    (finally
+      (.delete (java.io.File. "alldtypes-sparse.arrow")))))
+
+
+(deftest arrow-file-types
+  ;;lz4 compression
+  (let [all-files ["test/data/alldtypes.arrow-feather"            ;lz4
+                   "test/data/alldtypes.arrow-feather-compressed" ;zstd
+                   "test/data/alldtypes.arrow-feather-v1"         ;v1
+                   ]]
+    (doseq [file all-files]
+      (is (= 1000 (ds/row-count (arrow/stream->dataset file)))))
+    ;; lz4 with dependent frames))))))
+    (is (= 31962 (ds/row-count (arrow/stream->dataset "test/data/tweets_sentiment.feather"))))))
+
+
+(deftest base-ds-seq-test
+  (try
+    (let [ds (supported-datatype-ds)
+          _ (arrow/dataset-seq->stream! "alldtypes-seq.arrow" {:strings-as-text? false} [ds ds ds])
+          mmap-ds-seq (arrow/stream->dataset-seq "alldtypes-seq.arrow" {:key-fn keyword
+                                                                        :open-type :mmap})
+          copy-ds-seq (arrow/stream->dataset-seq "alldtypes-seq.arrow" {:key-fn keyword})]
+      (is (= 3 (count mmap-ds-seq)))
+      (is (= 3 (count copy-ds-seq)))
+      (let [mmap-ds (last mmap-ds-seq)
+            copy-ds (last copy-ds-seq)]
+        (doseq [col (vals ds)]
+         (let [cname ((meta col) :name)
+               dt (dtype/elemwise-datatype col)
+               inp-col (mmap-ds cname)
+               cp-col (copy-ds cname)]
+           (is (= dt (dtype/elemwise-datatype inp-col)) (str "inplace failure " cname))
+           (is (= dt (dtype/elemwise-datatype cp-col)) (str "copy failure " cname))
+           (is (= (vec col) (vec inp-col)) (str "inplace failure " cname))
+           (is (= (vec col) (vec cp-col)) (str "copy failure " cname))))))
+    (finally
+      (.delete (java.io.File. "alldtypes-seq.arrow")))))
+
+
+(deftest simple-stocks
+    (try
+      (let [stocks (ds/->dataset "test/data/stocks.csv")
+            _ (arrow/dataset->stream! stocks "temp.stocks.arrow")
+            stocks-copying (arrow/stream->dataset "temp.stocks.arrow")
+            stocks-inplace (arrow/stream->dataset "temp.stocks.arrow" {:open-type :mmap})
+            pystocks-copying (arrow/stream->dataset "test/data/stocks.pyarrow.stream")
+            pystocks-inplace (arrow/stream->dataset "test/data/stocks.pyarrow.stream")]
+        ;;This is here just to make sure that the data isn't cleaned up until it
+        ;;actually can safely be cleaned up.  This was a bug that caused datatype to
+        ;;bump from 5.11 to 5.12
+        (System/gc)
+        (is (dfn/equals (stocks "price") (stocks-copying "price")))
+        (is (dfn/equals (stocks "price") (stocks-inplace "price")))
+        (is (dfn/equals (stocks "price") (pystocks-copying "price")))
+        (is (dfn/equals (stocks "price") (pystocks-inplace "price")))
+
+        (is (= (vec (stocks "symbol")) (vec (stocks-copying "symbol"))))
+        (is (= (vec (stocks "symbol")) (vec (stocks-inplace "symbol"))))
+        ;;python saves strings inline in the file - equivalent to :strings-as-text?
+        ;;save option
+        (is (= (vec (stocks "symbol")) (mapv str (pystocks-copying "symbol"))))
+        (is (= (vec (stocks "symbol")) (mapv str (pystocks-inplace "symbol")))))
+      (finally
+        (.delete (java.io.File. "temp.stocks.arrow")))))
+
+
+(deftest ames-house-prices
+    (try
+      (let [ames (ds/->dataset "test/data/ames-house-prices/train.csv")
+            _ (arrow/dataset->stream! ames "temp.ames.arrow")
+            ames-copying (arrow/stream->dataset "temp.ames.arrow")
+            ames-inplace (arrow/stream->dataset "temp.ames.arrow" {:open-type :mmap})
+            pyames-copying (arrow/stream->dataset "test/data/ames.pyarrow.stream")
+            pyames-inplace (arrow/stream->dataset "test/data/ames.pyarrow.stream")]
+        (System/gc)
+        (is (dfn/equals (ames "SalePrice") (ames-copying "SalePrice")))
+        (is (dfn/equals (ames "SalePrice") (ames-inplace "SalePrice")))
+        (is (= (ds-col/missing (ames "LotFrontage"))
+               (ds-col/missing (ames-copying "LotFrontage"))))
+        (is (= (ds-col/missing (ames "LotFrontage"))
+               (ds-col/missing (ames-inplace "LotFrontage"))))
+        (is (not= 0 (dtype/ecount (ds-col/missing (ames-inplace "LotFrontage")))))
+        (is (dfn/equals (ames "SalePrice") (pyames-copying "SalePrice")))
+        (is (dfn/equals (ames "SalePrice") (pyames-inplace "SalePrice")))
+        (is (= (ds-col/missing (ames "LotFrontage"))
+               (ds-col/missing (pyames-copying "LotFrontage"))))
+        (is (= (ds-col/missing (ames "LotFrontage"))
+               (ds-col/missing (pyames-inplace "LotFrontage")))))
+      (finally
+        (.delete (java.io.File. "temp.ames.arrow")))))
+
+
+(deftest ames-compression-test
+  (try
+    (let [ames (ds/->dataset "test/data/ames-house-prices/train.csv")
+          _ (arrow/dataset->stream! ames "ames-uncompressed.arrow")
+          _ (arrow/dataset->stream! ames "ames-zstd.arrow" {:compression
+                                                            {:compression-type :zstd
+                                                             ;;default is 3
+                                                             :level 5}})
+          _ (arrow/dataset->stream! ames "ames-lz4.arrow" {:compression :lz4})
+          _ (arrow/dataset->stream! (sparse-col/->sparse-ds ames)
+                                    "ames-sparse-zstd.arrow" {:compression
+                                                              {:compression-type :zstd
+                                                               ;;default is 3
+                                                               :level 5}})
+          file-len (fn [path] (.length (java.io.File. (str path))))
+          _ (println (ds/->dataset {:save-type [:uncompressed :zstd :sparse-zstd :lz4]
+                                    :file-size [(file-len "ames-uncompressed.arrow")
+                                                (file-len "ames-zstd.arrow")
+                                                (file-len "ames-sparse-zstd.arrow")
+                                                (file-len "ames-lz4.arrow")]}))
+          uncomp (arrow/stream->dataset "ames-uncompressed.arrow")
+          zstd (arrow/stream->dataset "ames-zstd.arrow")
+          sparse-zstd (arrow/stream->dataset "ames-sparse-zstd.arrow")
+          lz4 (arrow/stream->dataset "ames-lz4.arrow")]
+      (System/gc)
+      (is (dfn/equals (uncomp "SalePrice") (zstd "SalePrice")))
+      (is (dfn/equals (uncomp "LotFrontage") (sparse-zstd "LotFrontage")))
+      (is (dfn/equals (uncomp "SalePrice") (lz4 "SalePrice"))))
+    (finally
+      (.delete (java.io.File. "ames-uncompressed.arrow"))
+      (.delete (java.io.File. "ames-zstd.arrow"))
+      (.delete (java.io.File. "ames-sparse-zstd.arrow"))
+      (.delete (java.io.File. "ames-lz4.arrow")))))
+
+
+(deftest date-arrow-test
+  (let [date-data (arrow/read-stream-dataset-copying "test/data/with_date.arrow"
+                                                     {:integer-datetime-types? true})]
+    (is (= [18070 18072 18063]
+           (date-data "date")))
+    (is (= :epoch-days (dtype/elemwise-datatype (date-data "date")))))
+  (let [date-data (arrow/read-stream-dataset-copying "test/data/with_date.arrow")]
+    (is (= (mapv #(java.time.LocalDate/parse %)
+                 ["2019-06-23" "2019-06-25" "2019-06-16"])
+           (date-data "date")))
+    (is (= :packed-local-date (dtype/elemwise-datatype (date-data "date"))))))
+
+(deftest odd-parquet-crash
+  (let [test-data (ds/->dataset "test/data/part-00000-74d3eb51-bc9c-4ba5-9d13-9e0d71eea31f.c000.snappy.parquet")]
+    (try
+      (arrow/write-dataset-to-stream! test-data "test.arrow")
+      (let [arrow-ds (arrow/read-stream-dataset-copying "test.arrow")]
+        (is (= (ds/missing test-data)
+               (ds/missing arrow-ds))))
+      (finally
+        (.delete (java.io.File. "test.arrow"))))))
+
+
+(deftest failed-R-file
+  (let [cp-data (arrow/read-stream-dataset-copying "test/data/part-8981.ipc_stream")
+        inp-data (arrow/read-stream-dataset-inplace "test/data/part-8981.ipc_stream")]
+    (is (= (vec (ds/column-names cp-data))
+           (vec (ds/column-names inp-data))))))
+
+
+(deftest large-var-char-file
+    (let [cp-data (arrow/read-stream-dataset-copying "test/data/largeVarChar.ipc")
+          inp-data (arrow/read-stream-dataset-inplace "test/data/largeVarChar.ipc")]
+      (is (= (vec (ds/column-names cp-data))
+             (vec (ds/column-names inp-data))))
+      (is (= (vec (first (ds/columns cp-data)))
+             (vec (first (ds/columns inp-data)))))))
+
+
+(deftest uuid-test
+  (let [py-uuid (ds/->dataset "test/data/uuid_ext.arrow" {:key-fn keyword})]
+    
+    (is (= :uuid (dtype/elemwise-datatype (py-uuid :id))))
+    (is (= (mapv #(java.util.UUID/fromString %)
+                 ["8be643d6-0df7-4e5e-837c-f94170c87914"
+                  "24bc9cf4-e2e8-444f-bb2d-82394f33ff76"
+                  "e8149e1b-aef6-4671-b1b4-3b7a21eed92a"])
+           (py-uuid :id))))
+  (try
+    (let [uuid-ds (ds/->dataset "test/data/uuid.parquet"
+                                {:parser-fn {"uuids" :uuid}})
+          _ (arrow/write-dataset-to-stream! uuid-ds "test-uuid.arrow")
+          copying-ds (arrow/read-stream-dataset-copying "test-uuid.arrow")
+          inplace-ds (arrow/read-stream-dataset-inplace "test-uuid.arrow")]
+      (is (= :uuid ((comp :datatype meta) (copying-ds "uuids"))))
+      (is (= :uuid ((comp :datatype meta) (inplace-ds "uuids"))))
+      (is (= (vec (copying-ds "uuids"))
+             (vec (inplace-ds "uuids"))))
+      (is (= (vec (uuid-ds "uuids"))
+             (vec (copying-ds "uuids")))))
+    (finally
+      (.delete (java.io.File. "test-uuid.arrow")))))
+
+
+(deftest local-time
+    (try
+      (let [ds (ds/->dataset {"a" (range 10)
+                              "b" (repeat 10 (java.time.LocalTime/now))})
+            _ (arrow/write-dataset-to-stream! ds "test-local-time.arrow")
+            copying-ds (arrow/read-stream-dataset-copying "test-local-time.arrow")
+            inplace-ds (arrow/read-stream-dataset-inplace "test-local-time.arrow")]
+        (is (= :packed-local-time (dtype/elemwise-datatype (copying-ds "b"))))
+        (is (= :packed-local-time (dtype/elemwise-datatype (inplace-ds "b"))))
+        (is (= (vec (copying-ds "b"))
+               (vec (inplace-ds "b"))))
+        ;;Making a primitive container will use the packed data.
+        (is (= (vec (ds "b"))
+               (vec (copying-ds "b")))))
+      (finally
+        (.delete (java.io.File. "test-local-time.arrow")))))
+
+
+(deftest string-arrow
+  (let [dataset (ds/->dataset [{"col1" "a"}] {:parser-fn :string})
+        baos (ByteArrayOutputStream.)]
+    (resource/stack-resource-context
+     (arrow/dataset->stream! dataset baos {:compression :lz4})
+     (let [written-bytes (.toByteArray baos)
+           arrow-ds-rtt (arrow/stream->dataset written-bytes)
+           _ (.reset baos)
+           _ (arrow/dataset->stream! arrow-ds-rtt baos {:compression :lz4})
+           b2 (.toByteArray baos)
+           final-ds (arrow/stream->dataset b2)]
+       (is (= (vec (dataset "col1"))
+              (vec (final-ds "col1"))))))))
+
+
+(deftest nullcol
+  (let [ds (arrow/stream->dataset "test/data/withnullcol.arrow")]
+    (is (= (vec (range (ds/row-count ds)))
+           (vec (ds/missing (ds "nullcol")))))))
+
+(deftest list-datatypes-read-only
+  (let [ds (ds/->dataset "test/data/arrow_list.arrow")]
+    (is (= [["dog" "car"]
+            ["dog" "flower"]
+            ["car" "flower"]]
+           (mapv vec (ds "class-name"))))))
+
+(deftest empty-array-dataset
+  (is (nil? (arrow/stream->dataset "test/data/empty.arrow"))))

tech.ml.dataset/test/tech/v3/libs/csv_test.clj

@@ -0,0 +1,28 @@
+(ns tech.v3.libs.csv-test
+  (:require [clojure.test :refer [deftest is testing]]
+            [tech.v3.dataset :as ds]
+            [tech.v3.dataset.io.csv :as csv-parse]))
+
+(def duplicate-headers-file "test/data/duplicate-headers.csv")
+
+(deftest ensure-unique-headers-test
+  (testing "that all headers are are forced to be unique"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true})]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7))
+    (let [ds (csv-parse/csv->dataset duplicate-headers-file
+                                     {:ensure-unique-column-names? true})]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7)))
+
+  (testing "that exception is thrown on duplicate headers"
+    (is (thrown? RuntimeException (ds/->dataset duplicate-headers-file))))
+
+  (testing "that custom postfix-fn works correctly"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true
+                            :unique-column-name-fn (fn [col-idx colname] (str colname "::" col-idx))})]
+      (is (some? (ds/column ds "column::2")))
+      (is (some? (ds/column ds "column::4")))
+      (is (some? (ds/column ds "column-1::6"))))))

tech.ml.dataset/test/tech/v3/libs/fastexcel_test.clj

@@ -0,0 +1,99 @@
+(ns tech.v3.libs.fastexcel-test
+  (:require [tech.v3.libs.fastexcel :as xlsx-parse]
+            [tech.v3.dataset :as ds]
+            [tech.v3.datatype :as dtype]
+            [clojure.test :refer [deftest is testing]]))
+
+(def xls-file "test/data/file_example_XLS_1000.xls")
+(def xlsx-file "test/data/file_example_XLSX_1000.xlsx")
+(def sparse-file "test/data/sparsefile.xlsx")
+(def stocks-file "test/data/stocks.xlsx")
+(def stocks-bad-date-file "test/data/stocks-bad-date.xlsx")
+(def duplicate-headers-file "test/data/duplicate-headers.xlsx")
+
+
+
+(deftest happy-path-parse-test
+  (let [ds (first (xlsx-parse/workbook->datasets xlsx-file))]
+    (is (= #{"column-0" "Age" "Country" "First Name" "Gender" "Date" "Last Name" "Id"}
+           (set (ds/column-names ds))))
+    (is (= #{:float64 :string}
+           (set (map dtype/get-datatype (ds/columns ds)))))
+    (is (= 1000 (ds/row-count ds)))
+    (is (= 8 (ds/column-count ds)))))
+
+
+
+(deftest sparse-file-parse-test
+  (let [ds (first (xlsx-parse/workbook->datasets sparse-file))]
+    (is (= 8 (ds/row-count ds)))
+    (is (= 8 (ds/column-count ds)))
+    (is (every? #(= (set (range 8)) %)
+                (map (comp set ds/missing ds) ["column-0" "a" "column-6"])))
+    (is (= [1.0 1.0 1.0 "a" 2.0 23.0]
+           (->> (ds/columns ds)
+                (mapcat (comp dtype/->reader ds/drop-missing))
+                vec)))))
+
+(deftest datetime-test
+  (let [ds (first (xlsx-parse/workbook->datasets
+                   stocks-file
+                   {:parser-fn {"date" :packed-local-date}}))]
+    (is (= :packed-local-date (dtype/get-datatype (ds "date"))))))
+
+
+(deftest bad-datetime-test
+  (let [ds (first (xlsx-parse/workbook->datasets stocks-bad-date-file))]
+    (is (= :string (dtype/get-datatype (ds "date"))))
+    (is (= {java.lang.String 29}
+           (->> (ds "date")
+                (map type)
+                frequencies)))))
+
+
+(deftest skip-rows-test
+  (let [ds (ds/->dataset "test/data/holdings-daily-us-en-mdy.xlsx"
+                         {:n-initial-skip-rows 4
+                          :parser-fn {"Identifier" :string
+                                      "Weight" :float64}})]
+    ;;column-8 had no data
+    (is (= #{:float64 :string :boolean}
+           (set (map dtype/get-datatype (vals ds)))))
+    (is (= ["Name"
+            "Ticker"
+            "Identifier"
+            "SEDOL"
+            "Weight"
+            "Sector"
+            "Shares Held"
+            "Local Currency"
+            "column-8"]
+           (vec (ds/column-names ds))))))
+
+(deftest ensure-unique-headers-test
+  (testing "that all headers are are forced to be unique"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true})]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7))
+    (let [ds (first (xlsx-parse/workbook->datasets duplicate-headers-file
+                                                   {:ensure-unique-column-names? true}))]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7)))
+
+  (testing "that exception is thrown on duplicate headers"
+    (is (thrown? RuntimeException (ds/->dataset duplicate-headers-file)))
+    (is (thrown? RuntimeException (xlsx-parse/workbook->datasets duplicate-headers-file))))
+
+  (testing "that custom postfix-fn works correctly"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true
+                            :unique-column-name-fn (fn [col-idx colname] (str colname "::" col-idx))})]
+      (is (some? (ds/column ds "column::2")))
+      (is (some? (ds/column ds "column::4")))
+      (is (some? (ds/column ds "column-1::6"))))))
+
+
+(deftest number-colname
+  (let [ds (ds/->dataset "test/data/number_column.xlsx")]
+    (is (= (first (ds/column-names ds)) 0.0))))

tech.ml.dataset/test/tech/v3/libs/parquet_test.clj

@@ -0,0 +1,124 @@
+(ns tech.v3.libs.parquet-test
+  (:require [tech.v3.dataset :as ds]
+            [tech.v3.datatype :as dtype]
+            [tech.v3.datatype.functional :as dfn]
+            [tech.v3.libs.parquet :as parquet]
+            [tech.v3.dataset.utils :as ds-utils]
+            [tech.v3.dataset.column :as ds-col]
+            [tech.v3.datatype.datetime :as dtype-dt]
+            [clojure.test :refer [deftest is]]))
+
+(ds-utils/set-slf4j-log-level :info)
+
+
+(deftest stocks-test
+  (try
+    (let [stocks (ds/->dataset "test/data/stocks.csv")
+          _ (ds/write! stocks "stocks.parquet")
+          stocks-p (ds/->dataset "stocks.parquet")]
+      (is (= (vec (stocks "symbol"))
+             (mapv str (stocks-p "symbol"))))
+      (is (dfn/equals (stocks "price")
+                      (stocks-p "price")))
+      (is (= (vec (stocks "date"))
+             (vec (stocks-p "date")))))
+    (finally
+      (.delete (java.io.File. "stocks.parquet")))))
+
+
+(deftest userdata1-test
+  (try
+    (let [testd (ds/->dataset "test/data/parquet/userdata1.parquet")
+          _ (ds/write! testd "userdata1.parquet")
+          newd (ds/->dataset "userdata1.parquet")
+          _ (ds/write! newd "userdata1.nippy")
+          nippy-d (ds/->dataset "userdata1.nippy")]
+      (is (= (vec (testd "registration_dttm"))
+             (vec (newd "registration_dttm"))))
+      (is (= (vec (testd "comments"))
+             (vec (newd "comments"))))
+      (is (= (vec (testd "comments"))
+             (vec (nippy-d "comments")))))
+
+    (finally
+      (.delete (java.io.File. "userdata1.parquet"))
+      (.delete (java.io.File. "userdata1.nippy")))))
+
+
+(deftest whitelist-test
+  (let [testd (ds/->dataset "test/data/parquet/userdata1.parquet"
+                            {:column-whitelist ["first_name" "last_name" "gender"]})]
+    (is (= 3 (ds/column-count testd)))))
+
+
+(deftest ames-ds
+  (try
+    (let [ames (ds/->dataset "test/data/ames-house-prices/train.csv")
+          _ (ds/write! ames "ames.parquet")
+          newd (ds/->dataset "ames.parquet")]
+      (is (= (ds/missing (ames "LotFrontage"))
+             (ds/missing (newd "LotFrontage"))))
+      (is (= (vec (ames "CentralAir"))
+             (vec (newd "CentralAir"))))
+      (is (dfn/equals (ames "SalePrice") (newd "SalePrice"))))
+    (finally
+      (.delete (java.io.File. "ames.parquet")))))
+
+
+(deftest uuid-test
+  (try
+    (let [uuid-ds (ds/->dataset "test/data/uuid.parquet"
+                                {:parser-fn {"uuids" :uuid}})
+          _ (ds/write! uuid-ds "test-uuid.parquet")
+          new-ds (ds/->dataset "test-uuid.parquet"
+                               {:parser-fn {"uuids" :uuid}})]
+      (is (= :uuid ((comp :datatype meta) (uuid-ds "uuids"))))
+      (is (= :uuid ((comp :datatype meta) (new-ds "uuids")))))
+    (finally
+      (.delete (java.io.File. "test-uuid.parquet")))))
+
+
+(deftest missing-uint8-data
+  ;;Use a large enough value the the system is forced to use uint8 columns else
+  ;;it will default to int8 columns based on the column data min/max
+  (let [ds (ds/->dataset {:a (dtype/make-container :uint8 [10 20 245])})
+        ds (ds/update-column ds :a #(ds-col/set-missing % [1 5]))]
+    (try
+      (parquet/ds->parquet ds "test.parquet")
+      (let [nds (ds/->dataset "test.parquet" {:key-fn keyword})]
+        (is (= 3 (ds/row-count nds)))
+        (is (= [1] (vec (dtype/->reader (ds/missing ds)))))
+        (is (= :uint8 (dtype/elemwise-datatype (ds :a))))
+        (is (= :uint8 (dtype/elemwise-datatype (nds :a))))
+        (is (= [1] (vec (dtype/->reader (ds/missing nds))))))
+      (finally
+        (.delete (java.io.File. "test.parquet"))))))
+
+
+(deftest nested-parquet
+  (let [ds (ds/->dataset "test/data/nested.parquet")]
+    (is (= [1 nil 2 nil 3 nil nil] (vec (ds "id"))))
+    (is (= ["a" "b" "a" "b" "a" "b" "c"] (vec (ds "val.key_value.key"))))
+    (is (= ["va" "vb" nil nil "vb" nil nil] (vec (ds "val2.key_value.key"))))))
+
+
+(deftest local-time
+  (try
+    (let [ds (ds/->dataset {:a (range 10)
+                            :b (repeat 10 (java.time.LocalTime/now))})
+          _ (parquet/ds->parquet ds "test.parquet")
+          pds (ds/->dataset "test.parquet"  {:key-fn keyword})]
+      (is (= (vec (ds :b))
+             (vec (pds :b)))))
+    (finally
+      (.delete (java.io.File. "test.parquet")))))
+
+
+(deftest decimaltable
+  (let [table (ds/->dataset "test/data/decimaltable.parquet")
+        decimals (table "decimals")]
+    (is (dfn/equals [3.420 1.246] decimals))))
+
+
+(deftest issue-401-paruet-missing-column
+  (is (= 4 (ds/column-count (ds/->dataset "test/data/2024-03-03.parquet")))))

tech.ml.dataset/test/tech/v3/libs/poi_test.clj

@@ -0,0 +1,115 @@
+(ns tech.v3.libs.poi-test
+  (:require [tech.v3.libs.poi :as xlsx-parse]
+            [tech.v3.dataset :as ds]
+            [tech.v3.dataset.column :as ds-col]
+            [tech.v3.datatype.functional :as dfn]
+            [tech.v3.datatype :as dtype]
+            [clojure.test :refer [deftest is testing]]))
+
+
+(def xls-file "test/data/file_example_XLS_1000.xls")
+(def xlsx-file "test/data/file_example_XLSX_1000.xlsx")
+(def sparse-file "test/data/sparsefile.xlsx")
+(def stocks-file "test/data/stocks.xlsx")
+(def duplicate-headers-file "test/data/duplicate-headers.xls")
+
+
+(deftest happy-path-parse-test
+  (let [ds (first (xlsx-parse/workbook->datasets xlsx-file))
+        ds2 (first (xlsx-parse/workbook->datasets xlsx-file))]
+    (is (= #{"column-0" "Age" "Country" "First Name" "Gender" "Date" "Last Name" "Id"}
+           (set (ds/column-names ds))))
+    (is (= #{"column-0" "Age" "Country" "First Name" "Gender" "Date" "Last Name" "Id"}
+           (set (ds/column-names ds2))))
+    (is (= #{:float64 :string}
+           (set (map dtype/get-datatype (ds/columns ds)))))
+    (is (= 1000 (ds/row-count ds)))
+    (is (= 1000 (ds/row-count ds2)))
+    (is (= 8 (ds/column-count ds)))
+    (is (= 8 (ds/column-count ds2)))
+    (is (dfn/equals (ds "Age") (ds2 "Age")))
+    (is (dfn/equals (ds "Id") (ds2 "Id")))))
+
+
+(deftest sparse-file-parse-test
+  (let [ds (first (xlsx-parse/workbook->datasets sparse-file))]
+    (is (= 8 (ds/row-count ds)))
+    (is (= 8 (ds/column-count ds)))
+    (is (every? #(= (set (range 8)) %)
+                (map (comp set ds-col/missing ds) ["column-0" "a" "column-6"])))
+    (is (= [1.0 1.0 1.0 "a" 2.0 23.0]
+           (->> (ds/columns ds)
+                (mapcat (comp dtype/->reader ds/drop-missing))
+                vec)))))
+
+
+(deftest datetime-test
+  (let [ds (first (xlsx-parse/workbook->datasets
+                   stocks-file
+                   {:parser-fn {"date" :packed-local-date}}))]
+    (is (= :packed-local-date (dtype/get-datatype (ds "date"))))))
+
+
+(deftest custom-parser-test
+  (let [ds (first (xlsx-parse/workbook->datasets
+                   xls-file
+                   {:parser-fn {"Date" [:local-date
+                                        "dd/MM/yyyy"]}}))]
+    (is (= :local-date (dtype/get-datatype (ds "Date"))))))
+
+
+(deftest integer-field-test
+  (let [ds (first (xlsx-parse/workbook->datasets
+                   xls-file
+                   {:parser-fn {"Id" :int64}}))]
+    (is (= :int64 (dtype/get-datatype (ds "Id"))))))
+
+
+(deftest xls-keyword-colnames
+  (let [ds (first (xlsx-parse/workbook->datasets
+                   xls-file
+                   {:key-fn keyword}))]
+    ;;The first column is an integer so keyword returns nil for that.
+    ;;This is also a good example in that the system produces keywords with spaces
+    ;;in them...that definitely isn't ideal.
+    (is (every? keyword? (rest (ds/column-names ds))))))
+
+
+(deftest key-fn-number-columns
+  (let [ds (first (xlsx-parse/workbook->datasets xlsx-file {:key-fn keyword}))]
+    (is (= 0 (count (filter nil? (ds/column-names ds)))))
+    (is (= #{:column-0 :Age :Country (keyword "First Name") :Gender :Date
+             (keyword "Last Name") (keyword "Id")}
+           (set (ds/column-names ds))))))
+
+
+(deftest auto-infer-dates
+  (let [ds (first (xlsx-parse/workbook->datasets "test/data/stocks-with-dates.xlsx"))]
+    (is (= #{:string :packed-local-date :float64}
+           (->> (vals ds)
+                (map (comp :datatype meta))
+                set)))))
+
+
+(deftest ensure-unique-headers-test
+  (testing "that all headers are are forced to be unique"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true})]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7))
+    (let [ds (first (xlsx-parse/workbook->datasets duplicate-headers-file
+                                                   {:ensure-unique-column-names? true}))]
+      (is (ds/column-count ds) 7)
+      (is (count (set (ds/column-names ds))) 7)))
+
+  (testing "that exception is thrown on duplicate headers"
+    (is (thrown? RuntimeException (ds/->dataset duplicate-headers-file)))
+    (is (thrown? RuntimeException (xlsx-parse/workbook->datasets duplicate-headers-file))))
+
+  (testing "that custom postfix-fn works correctly"
+    (let [ds (ds/->dataset duplicate-headers-file
+                           {:ensure-unique-column-names? true
+                            :unique-column-name-fn (fn [col-idx colname] (str colname "::" col-idx))})]
+      (is (some? (ds/column ds "column::2")))
+      (is (some? (ds/column ds "column::4")))
+      (is (some? (ds/column ds "column-1::6"))))))