init research

tech.ml.dataset/test/tech/v3/dataset/github_test.clj

@@ -0,0 +1,37 @@
+(ns tech.v3.dataset.github-test
+  (:require [tech.v3.dataset :as ds]
+            [tech.v3.datatype :as dtype]
+            [clojure.test :refer [deftest is]]))
+
+
+
+(comment
+  ;;This sometimes returns a 500 error.
+(deftest load-github-events
+  (let [ds (ds/->dataset "https://api.github.com/events" {:file-type :json
+                                                          :key-fn keyword})]
+    (is (every? keyword? (ds/column-names ds)))
+    (is (= [8 30] (dtype/shape ds)))))
+(do
+  (require '[tech.v3.datatype.functional :as dfn])
+  (require '[tech.v3.datatype.argops :as argops])
+  (require '[tech.v3.datatype.unary-pred :as un-pred])
+  (defonce flights (ds/->dataset "https://raw.githubusercontent.com/Rdatatable/data.table/master/vignettes/flights14.csv")))
+
+(time (-> (dfn/+ (flights "arr_delay")
+                 (flights "dep_delay"))
+          (dfn/< 0)
+          (un-pred/bool-reader->indexes)
+          (dtype/ecount)))
+
+;;Another way to get the same result is to use summation.  Booleans are
+;;interpreted very specifically below where false is 0 and 1 is true.
+;;Double summation is very fast.
+(time (-> (dfn/+ (flights "arr_delay")
+                 (flights "dep_delay"))
+          (dfn/< 0)
+          (dfn/sum)))
+
+
+
+  )