796 lines
39 KiB
HTML
Vendored
796 lines
39 KiB
HTML
Vendored
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
|
<!-- NewPage -->
|
|
<html lang="en">
|
|
<head>
|
|
<!-- Generated by javadoc (1.8.0_312) on Sun Dec 11 09:30:03 MST 2022 -->
|
|
<title>Reductions (tech.ml.dataset Documentation)</title>
|
|
<meta name="date" content="2022-12-11">
|
|
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">
|
|
<script type="text/javascript" src="../../../script.js"></script>
|
|
</head>
|
|
<body>
|
|
<script type="text/javascript"><!--
|
|
try {
|
|
if (location.href.indexOf('is-external=true') == -1) {
|
|
parent.document.title="Reductions (tech.ml.dataset Documentation)";
|
|
}
|
|
}
|
|
catch(err) {
|
|
}
|
|
//-->
|
|
var methods = {"i0":9,"i1":9,"i2":9,"i3":9,"i4":9,"i5":9,"i6":9,"i7":9,"i8":9,"i9":9,"i10":9,"i11":9,"i12":9,"i13":9,"i14":9,"i15":9,"i16":9,"i17":9,"i18":9,"i19":9,"i20":9,"i21":9,"i22":9,"i23":9,"i24":9};
|
|
var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],8:["t4","Concrete Methods"]};
|
|
var altColor = "altColor";
|
|
var rowColor = "rowColor";
|
|
var tableTab = "tableTab";
|
|
var activeTableTab = "activeTableTab";
|
|
</script>
|
|
<noscript>
|
|
<div>JavaScript is disabled on your browser.</div>
|
|
</noscript>
|
|
<!-- ========= START OF TOP NAVBAR ======= -->
|
|
<div class="topNav"><a name="navbar.top">
|
|
<!-- -->
|
|
</a>
|
|
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
|
|
<a name="navbar.top.firstrow">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="navList" title="Navigation">
|
|
<li><a href="../../../overview-summary.html">Overview</a></li>
|
|
<li><a href="package-summary.html">Package</a></li>
|
|
<li class="navBarCell1Rev">Class</li>
|
|
<li><a href="package-tree.html">Tree</a></li>
|
|
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
|
|
<li><a href="../../../index-all.html">Index</a></li>
|
|
<li><a href="../../../help-doc.html">Help</a></li>
|
|
</ul>
|
|
<div class="aboutLanguage"><script async src="https://www.googletagmanager.com/gtag/js?id=G-RGTB4J7LGP"</script><script>window.dataLayer = window.dataLayer || [];\nfunction gtag(){dataLayer.push(arguments);}\ngtag('js', new Date());\ngtag('config', 'G-RGTB4J7LGP');</div>
|
|
</div>
|
|
<div class="subNav">
|
|
<ul class="navList">
|
|
<li><a href="../../../tech/v3/dataset/Modelling.html" title="class in tech.v3.dataset"><span class="typeNameLink">Prev Class</span></a></li>
|
|
<li><a href="../../../tech/v3/dataset/Rolling.html" title="class in tech.v3.dataset"><span class="typeNameLink">Next Class</span></a></li>
|
|
</ul>
|
|
<ul class="navList">
|
|
<li><a href="../../../index.html?tech/v3/dataset/Reductions.html" target="_top">Frames</a></li>
|
|
<li><a href="Reductions.html" target="_top">No Frames</a></li>
|
|
</ul>
|
|
<ul class="navList" id="allclasses_navbar_top">
|
|
<li><a href="../../../allclasses-noframe.html">All Classes</a></li>
|
|
</ul>
|
|
<div>
|
|
<script type="text/javascript"><!--
|
|
allClassesLink = document.getElementById("allclasses_navbar_top");
|
|
if(window==top) {
|
|
allClassesLink.style.display = "block";
|
|
}
|
|
else {
|
|
allClassesLink.style.display = "none";
|
|
}
|
|
//-->
|
|
</script>
|
|
</div>
|
|
<div>
|
|
<ul class="subNavList">
|
|
<li>Summary: </li>
|
|
<li>Nested | </li>
|
|
<li>Field | </li>
|
|
<li>Constr | </li>
|
|
<li><a href="#method.summary">Method</a></li>
|
|
</ul>
|
|
<ul class="subNavList">
|
|
<li>Detail: </li>
|
|
<li>Field | </li>
|
|
<li>Constr | </li>
|
|
<li><a href="#method.detail">Method</a></li>
|
|
</ul>
|
|
</div>
|
|
<a name="skip.navbar.top">
|
|
<!-- -->
|
|
</a></div>
|
|
<!-- ========= END OF TOP NAVBAR ========= -->
|
|
<!-- ======== START OF CLASS DATA ======== -->
|
|
<div class="header">
|
|
<div class="subTitle">tech.v3.dataset</div>
|
|
<h2 title="Class Reductions" class="title">Class Reductions</h2>
|
|
</div>
|
|
<div class="contentContainer">
|
|
<ul class="inheritance">
|
|
<li>java.lang.Object</li>
|
|
<li>
|
|
<ul class="inheritance">
|
|
<li>tech.v3.dataset.Reductions</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<div class="description">
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<hr>
|
|
<br>
|
|
<pre>public class <span class="typeNameLabel">Reductions</span>
|
|
extends java.lang.Object</pre>
|
|
<div class="block"><p>High speed grouping aggregations based on sequences of datasets.</p></div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="summary">
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<!-- ========== METHOD SUMMARY =========== -->
|
|
<ul class="blockList">
|
|
<li class="blockList"><a name="method.summary">
|
|
<!-- -->
|
|
</a>
|
|
<h3>Method Summary</h3>
|
|
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
|
|
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd"> </span></span><span id="t1" class="tableTab"><span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd"> </span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd"> </span></span></caption>
|
|
<tr>
|
|
<th class="colFirst" scope="col">Modifier and Type</th>
|
|
<th class="colLast" scope="col">Method and Description</th>
|
|
</tr>
|
|
<tr id="i0" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#distinct-java.lang.Object-">distinct</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a distinct reducer produces a set of distinct values.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i1" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#distinctUInt32-java.lang.Object-">distinctUInt32</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a distinct reducer that produces a roaringbitmap of distinct values.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i2" class="altColor">
|
|
<td class="colFirst"><code>static java.util.Map</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#groupByColumnsAgg-java.lang.Iterable-java.lang.Object-java.util.Map-java.util.Map-">groupByColumnsAgg</a></span>(java.lang.Iterable dsSeq,
|
|
java.lang.Object colname,
|
|
java.util.Map aggMap,
|
|
java.util.Map options)</code>
|
|
<div class="block">Group a sequence of datasets by column or columns an in the process perform an aggregation.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i3" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#mean-java.lang.Object-">mean</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a mean reducer that produces a mean value of an individual source column.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i4" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probCDFS-java.lang.Object-java.lang.Object-">probCDFS</a></span>(java.lang.Object colname,
|
|
java.lang.Object cdfs)</code>
|
|
<div class="block">Probabilistic CDF calculation, one for each double cdf passed in.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i5" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probCDFS-java.lang.Object-java.lang.Object-long-">probCDFS</a></span>(java.lang.Object colname,
|
|
java.lang.Object cdfs,
|
|
long k)</code>
|
|
<div class="block">Probabilistic CDF calculation, one for each double cdf passed in.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i6" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probInterquartileRange-java.lang.Object-">probInterquartileRange</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Probabilistic interquartile range.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i7" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probInterquartileRange-java.lang.Object-long-">probInterquartileRange</a></span>(java.lang.Object colname,
|
|
long k)</code>
|
|
<div class="block">Probabilistic interquartile range.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i8" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probMedian-java.lang.Object-">probMedian</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Probabilistic median with default K of 128.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i9" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probMedian-java.lang.Object-long-">probMedian</a></span>(java.lang.Object colname,
|
|
long k)</code>
|
|
<div class="block">Probabilistic median.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i10" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probPMFS-java.lang.Object-java.lang.Object-">probPMFS</a></span>(java.lang.Object colname,
|
|
java.lang.Object pmfs)</code>
|
|
<div class="block">Returns an approximation to the Probability Mass Function (PMF) of the input stream given a set of splitPoints (values).</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i11" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probPMFS-java.lang.Object-java.lang.Object-long-">probPMFS</a></span>(java.lang.Object colname,
|
|
java.lang.Object pmfs,
|
|
long k)</code>
|
|
<div class="block">Returns an approximation to the Probability Mass Function (PMF) of the input stream given a set of splitPoints (values).</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i12" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probQuantile-java.lang.Object-double-">probQuantile</a></span>(java.lang.Object colname,
|
|
double quantile)</code>
|
|
<div class="block">Probabilistic quantile estimation using default k of 128.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i13" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probQuantile-java.lang.Object-double-long-">probQuantile</a></span>(java.lang.Object colname,
|
|
double quantile,
|
|
long k)</code>
|
|
<div class="block">Probabilistic quantile estimation using default k of 128.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i14" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probQuantiles-java.lang.Object-java.lang.Object-">probQuantiles</a></span>(java.lang.Object colname,
|
|
java.lang.Object quantiles)</code>
|
|
<div class="block">Probabilistic quantile estimation using default k of 128.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i15" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probQuantiles-java.lang.Object-java.lang.Object-long-">probQuantiles</a></span>(java.lang.Object colname,
|
|
java.lang.Object quantiles,
|
|
long k)</code>
|
|
<div class="block">Probabilistic quantile estimation - see <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i16" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#probSetCardinality-java.lang.Object-java.util.Map-">probSetCardinality</a></span>(java.lang.Object colname,
|
|
java.util.Map options)</code>
|
|
<div class="block">Calculate a probabilistic set cardinality for a given column based on one of three algorithms.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i17" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#reducer-java.lang.Object-clojure.lang.IFn-">reducer</a></span>(java.lang.Object colname,
|
|
clojure.lang.IFn perElemFn)</code>
|
|
<div class="block">Create a custom reducer.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i18" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#reducer-java.lang.Object-clojure.lang.IFn-clojure.lang.IFn-">reducer</a></span>(java.lang.Object colname,
|
|
clojure.lang.IFn perElemFn,
|
|
clojure.lang.IFn finalizeFn)</code>
|
|
<div class="block">Create a custom reducer.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i19" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#reservoirDataset-long-">reservoirDataset</a></span>(long nRows)</code>
|
|
<div class="block">Return a reducer that produces a probabilistically sampled dataset of at most nRows len.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i20" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#reservoirStats-java.lang.Object-long-java.lang.Object-">reservoirStats</a></span>(java.lang.Object colname,
|
|
long nRows,
|
|
java.lang.Object statName)</code>
|
|
<div class="block">Return a reducer which will probabilistically sample the source column producing at most nRows and then call descriptiveStatistics on it with statName.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i21" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#rowCount-java.lang.Object-">rowCount</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a rowCount reducer that returns the number of source rows aggregated.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i22" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#setCardinality-java.lang.Object-">setCardinality</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a distinct reducer returns the number of distinct elements.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i23" class="rowColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#setCardinalityUint32-java.lang.Object-">setCardinalityUint32</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a distinct reducer that expects unsigned integer values and returns the number of distinct elements.</div>
|
|
</td>
|
|
</tr>
|
|
<tr id="i24" class="altColor">
|
|
<td class="colFirst"><code>static java.lang.Object</code></td>
|
|
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/dataset/Reductions.html#sum-java.lang.Object-">sum</a></span>(java.lang.Object colname)</code>
|
|
<div class="block">Returns a summation reducer that sums an individual source column.</div>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
<ul class="blockList">
|
|
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
|
|
<!-- -->
|
|
</a>
|
|
<h3>Methods inherited from class java.lang.Object</h3>
|
|
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="details">
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<!-- ============ METHOD DETAIL ========== -->
|
|
<ul class="blockList">
|
|
<li class="blockList"><a name="method.detail">
|
|
<!-- -->
|
|
</a>
|
|
<h3>Method Detail</h3>
|
|
<a name="groupByColumnsAgg-java.lang.Iterable-java.lang.Object-java.util.Map-java.util.Map-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>groupByColumnsAgg</h4>
|
|
<pre>public static java.util.Map groupByColumnsAgg(java.lang.Iterable dsSeq,
|
|
java.lang.Object colname,
|
|
java.util.Map aggMap,
|
|
java.util.Map options)</pre>
|
|
<div class="block"><p>Group a sequence of datasets by column or columns an in the process perform an aggregation. The resulting dataset will have one row per grouped key. Columns used as keys will always be represented in the result.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>dsSeq</code> - Sequence of datasets such as produced by rowMapcat, dsPmap, or loading many files.</dd>
|
|
<dd><code>colname</code> - Either a single column name or a vector of column names. These will be the grouping keys.</dd>
|
|
<dd><code>aggMap</code> - Map of result colname to reducer. Various reducers are provided or you can build your own via the <code>reducer</code> function.</dd>
|
|
<dd><code>options</code> - Options map. Described below. May be null.</p>
|
|
<p>Options:</p>
|
|
<ul>
|
|
<li><code>:map-initial-capacity</code> - initial hashmap capacity. Resizing hash-maps is expensive so we would like to set this to something reasonable. Defaults to 100000.</li>
|
|
<li><code>:index-filter</code> - A function that given a dataset produces a function from long index to boolean. Only indexes for which the index-filter returns true will be added to the aggregation. For very large datasets, this is a bit faster than using filter before the aggregation.</li>
|
|
</ul>
|
|
<p>Example:</p>
|
|
<pre><code class="java">//Begin parallelized expansion
|
|
Iterable dsSeq = (Iterable)rowMapcat(srcds, tallyDays, hashmap(kw("result-type"), kw("as-seq")));
|
|
|
|
//The first aggregation is to summarize by placement and simulation the year-month tallies.
|
|
//We are essentially replacing count with a summarized count. After this statement
|
|
//we can guarantee that the dataset has unique tuples of [simulation, placement, year-month]
|
|
Map initAgg = Reductions.groupByColumnsAgg(dsSeq, vector("simulation", "placement", "year-month"),
|
|
hashmap("count", Reductions.sum("count")),
|
|
null);
|
|
println(head(initAgg));
|
|
//["simulation" "placement" "year-month"]-aggregation [5 4]:
|
|
|
|
//| simulation | placement | year-month | count |
|
|
//|-----------:|----------:|------------|------:|
|
|
//| 0 | 0 | 2020-12 | 622.0 |
|
|
//| 0 | 1 | 2020-12 | 591.0 |
|
|
//| 0 | 2 | 2020-12 | 500.0 |
|
|
//| 0 | 3 | 2020-12 | 549.0 |
|
|
//| 0 | 4 | 2020-12 | 595.0 |
|
|
|
|
// The second aggregation allows us to build of statistics over each placement/year-month
|
|
// pair thus finding out the distribution of a given placement, year-month across simluations
|
|
Map result = Reductions.groupByColumnsAgg(vector(initAgg), vector("placement", "year-month"),
|
|
hashmap("min-count", Reductions.probQuantile("count", 0.0),
|
|
"low-95-count", Reductions.probQuantile("count", 0.05),
|
|
"q1-count", Reductions.probQuantile("count", 0.25),
|
|
"median-count", Reductions.probQuantile("count", 0.5),
|
|
"q3-count", Reductions.probQuantile("count", 0.75),
|
|
"high-95-count", Reductions.probQuantile("count", 0.95),
|
|
"max-count", Reductions.probQuantile("count", 1.0),
|
|
"count", Reductions.sum("count")),
|
|
null);
|
|
//Take a million row dataset, expand it, then perform two grouping aggregations.
|
|
println(head(result));
|
|
//["placement" "year-month"]-aggregation [5 10]:
|
|
|
|
//| q3-count | median-count | min-count | high-95-count | placement | max-count | count | low-95-count | q1-count | year-month |
|
|
//|---------:|-------------:|----------:|--------------:|----------:|----------:|--------:|-------------:|---------:|------------|
|
|
//| 646.0 | 593.0 | 366.0 | 716.0 | 36 | 809.0 | 58920.0 | 475.0 | 536.0 | 2020-12 |
|
|
//| 621.0 | 560.0 | 376.0 | 739.0 | 36 | 782.0 | 57107.0 | 459.0 | 512.0 | 2020-10 |
|
|
//| 168.0 | 139.0 | 25.0 | 211.0 | 0 | 246.0 | 13875.0 | 76.0 | 112.0 | 2021-01 |
|
|
//| 658.0 | 607.0 | 384.0 | 745.0 | 0 | 825.0 | 60848.0 | 486.0 | 561.0 | 2020-12 |
|
|
//| 628.0 | 581.0 | 422.0 | 693.0 | 0 | 802.0 | 58148.0 | 468.0 | 539.0 | 2020-11 |
|
|
</code></pre></dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="reducer-java.lang.Object-clojure.lang.IFn-clojure.lang.IFn-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>reducer</h4>
|
|
<pre>public static java.lang.Object reducer(java.lang.Object colname,
|
|
clojure.lang.IFn perElemFn,
|
|
clojure.lang.IFn finalizeFn)</pre>
|
|
<div class="block"><p>Create a custom reducer. perElemFn is passed the last return value as the first argument followed by a value from each column as additional arguments. It must always return the current context.</p>
|
|
<p>This is a easy way to instantiate tech.v3.datatype.IndexReduction so if you really need the best possible performance you need to implement three methods of IndexReduction:</p>
|
|
<ul>
|
|
<li><code>prepareBatch</code> - Passed each dataset before processing. Return value becomes first argument to <code>reduceIndex</code>.</li>
|
|
<li><code>reduceIndex</code> - Passed batchCtx, valCtx, and rowIdx. Must return an updated or new valCtx.</li>
|
|
<li><code>finalize</code> - Passed valCtx and must return the final per-row value expected in result dataset. The default is just to return valCtx.</li>
|
|
</ul>
|
|
<p>For <code>groupByColumnAgg</code> you do not need to worry about reduceReductions - there is no merge step.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>colname</code> - One or more column names. If multiple column names are specified then perElemFn will need to take additional arguments.</dd>
|
|
<dd><code>perElemFn</code> - A function that takes the previous context along with the current row’s column values and returns a new context.</dd>
|
|
<dd><code>finalizeFn</code> - Optional function that performs a final calculation taking a context and returning a value.</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="reducer-java.lang.Object-clojure.lang.IFn-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>reducer</h4>
|
|
<pre>public static java.lang.Object reducer(java.lang.Object colname,
|
|
clojure.lang.IFn perElemFn)</pre>
|
|
<div class="block"><p>Create a custom reducer. <code>perElemFn</code> is passed the last return value as the first argument followed by a value from each column as additional arguments. It must always return the current context.</p>
|
|
<p>This is a easy way to instantiate tech.v3.datatype.IndexReduction so if you really need the best possible performance you need to implement three methods of IndexReduction:</p>
|
|
<ul>
|
|
<li><code>prepareBatch</code> - Passed each dataset before processing. Return value becomes first argument to <code>reduceIndex</code>.</li>
|
|
<li><code>reduceIndex</code> - Passed batchCtx, valCtx, and rowIdx. Must return valCtx.</li>
|
|
<li><code>finalize</code> - Passed valCtx and must return the final per-row value expected in result dataset.</li>
|
|
</ul>
|
|
<p>For <code>groupByColumnAgg</code> you do not need to worry about reduceReductions - there is no merge step.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>colname</code> - One or more column names. If multiple column names are specified then perElemFn will need to take additional arguments.</dd>
|
|
<dd><code>perElemFn</code> - A function that takes the previous context along with the current row’s column values and returns a new context.</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="sum-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>sum</h4>
|
|
<pre>public static java.lang.Object sum(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a summation reducer that sums an individual source column.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="mean-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>mean</h4>
|
|
<pre>public static java.lang.Object mean(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a mean reducer that produces a mean value of an individual source column.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="rowCount-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>rowCount</h4>
|
|
<pre>public static java.lang.Object rowCount(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a rowCount reducer that returns the number of source rows aggregated.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="distinct-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>distinct</h4>
|
|
<pre>public static java.lang.Object distinct(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a distinct reducer produces a set of distinct values.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="distinctUInt32-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>distinctUInt32</h4>
|
|
<pre>public static java.lang.Object distinctUInt32(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a distinct reducer that produces a roaringbitmap of distinct values. This is many times faster than the distinct reducer if your data fits into unsigned int32 space.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="setCardinality-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>setCardinality</h4>
|
|
<pre>public static java.lang.Object setCardinality(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a distinct reducer returns the number of distinct elements.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="setCardinalityUint32-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>setCardinalityUint32</h4>
|
|
<pre>public static java.lang.Object setCardinalityUint32(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Returns a distinct reducer that expects unsigned integer values and returns the number of distinct elements. This is many times faster than the countDistinct function.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="reservoirDataset-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>reservoirDataset</h4>
|
|
<pre>public static java.lang.Object reservoirDataset(long nRows)</pre>
|
|
<div class="block"><p>Return a reducer that produces a probabilistically sampled dataset of at most nRows len.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="reservoirStats-java.lang.Object-long-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>reservoirStats</h4>
|
|
<pre>public static java.lang.Object reservoirStats(java.lang.Object colname,
|
|
long nRows,
|
|
java.lang.Object statName)</pre>
|
|
<div class="block"><p>Return a reducer which will probabilistically sample the source column producing at most nRows and then call descriptiveStatistics on it with statName.</p>
|
|
<p>Stat names are described in tech.v3.datatype.Statistics.descriptiveStats.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probSetCardinality-java.lang.Object-java.util.Map-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probSetCardinality</h4>
|
|
<pre>public static java.lang.Object probSetCardinality(java.lang.Object colname,
|
|
java.util.Map options)</pre>
|
|
<div class="block"><p>Calculate a probabilistic set cardinality for a given column based on one of three algorithms.</p>
|
|
<p>Options:</p>
|
|
<ul>
|
|
<li><code>:datatype</code> - One of <code>#{:float64 :string}</code>. Unspecified defaults to <code>:float64</code>.</li>
|
|
<li><code>:algorithm</code> - defaults to :hyper-log-log. Further algorithm-specific options may be included in the options map.</li>
|
|
</ul>
|
|
<p>Algorithm specific options:</p>
|
|
<ul>
|
|
<li><a href="https://datasketches.apache.org/docs/HLL/HLL.html">:hyper-log-log</a>
|
|
<ul>
|
|
<li><code>:hll-lgk</code> - defaults to 12, this is log-base2 of k, so k = 4096. lgK can be from 4 to 21.</li>
|
|
<li><code>:hll-type</code> - One of #{4,6,8}, defaults to 8. The HLL_4, HLL_6 and HLL_8 represent different levels of compression of the final HLL array where the 4, 6 and 8 refer to the number of bits each bucket of the HLL array is compressed down to. The HLL_4 is the most compressed but generally slightly slower than the other two, especially during union operations.</li>
|
|
</ul>
|
|
</li>
|
|
<li><a href="https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html">:theta</a></li>
|
|
<li><a href="https://datasketches.apache.org/docs/CPC/CPC.html">:cpc</a>
|
|
<ul>
|
|
<li><code>:cpc-lgk</code> - Defaults to 10.</li>
|
|
</ul>
|
|
</li>
|
|
</ul></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probQuantiles-java.lang.Object-java.lang.Object-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probQuantiles</h4>
|
|
<pre>public static java.lang.Object probQuantiles(java.lang.Object colname,
|
|
java.lang.Object quantiles,
|
|
long k)</pre>
|
|
<div class="block"><p>Probabilistic quantile estimation - see <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>quantiles</code> - Sequence of quantiles.</dd>
|
|
<dd><code>k</code> - Defaults to 128. This produces a normalized rank error of about 1.7%"</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="probQuantiles-java.lang.Object-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probQuantiles</h4>
|
|
<pre>public static java.lang.Object probQuantiles(java.lang.Object colname,
|
|
java.lang.Object quantiles)</pre>
|
|
<div class="block"><p>Probabilistic quantile estimation using default k of 128. See <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>quantiles</code> - Sequence of numbers from 0-1.</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="probQuantile-java.lang.Object-double-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probQuantile</h4>
|
|
<pre>public static java.lang.Object probQuantile(java.lang.Object colname,
|
|
double quantile,
|
|
long k)</pre>
|
|
<div class="block"><p>Probabilistic quantile estimation using default k of 128. See <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>. Multiple quantile calculations on a single source column will be merged into a single quantile calculation so it may be more convenient to use this function to produce multiple quantiles mapped to several result columns as opposed to ending up with a single column of maps of quantile to value.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>quantile</code> - Number from 0-1.</dd>
|
|
<dd><code>k</code> - Defaults to 128. This produces a normalized rank error of about 1.7%</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="probQuantile-java.lang.Object-double-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probQuantile</h4>
|
|
<pre>public static java.lang.Object probQuantile(java.lang.Object colname,
|
|
double quantile)</pre>
|
|
<div class="block"><p>Probabilistic quantile estimation using default k of 128. See <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>. Multiple quantiles will be merged into a single quantile calculation so it may be more convenient to use this function to produce multiple quantiles mapped to several result columns as opposed to ending up with a single column of maps of quantile to value.</p></div>
|
|
<dl>
|
|
<dt><span class="paramLabel">Parameters:</span></dt>
|
|
<dd><code>quantile</code> - Number from 0-1.</dd>
|
|
</dl>
|
|
</li>
|
|
</ul>
|
|
<a name="probMedian-java.lang.Object-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probMedian</h4>
|
|
<pre>public static java.lang.Object probMedian(java.lang.Object colname,
|
|
long k)</pre>
|
|
<div class="block"><p>Probabilistic median. See documentation for probQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probMedian-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probMedian</h4>
|
|
<pre>public static java.lang.Object probMedian(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Probabilistic median with default K of 128. See documentation for probQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probInterquartileRange-java.lang.Object-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probInterquartileRange</h4>
|
|
<pre>public static java.lang.Object probInterquartileRange(java.lang.Object colname,
|
|
long k)</pre>
|
|
<div class="block"><p>Probabilistic interquartile range. See documentation for probQuantile.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probInterquartileRange-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probInterquartileRange</h4>
|
|
<pre>public static java.lang.Object probInterquartileRange(java.lang.Object colname)</pre>
|
|
<div class="block"><p>Probabilistic interquartile range. See documentation for probQuantile.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probCDFS-java.lang.Object-java.lang.Object-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probCDFS</h4>
|
|
<pre>public static java.lang.Object probCDFS(java.lang.Object colname,
|
|
java.lang.Object cdfs,
|
|
long k)</pre>
|
|
<div class="block"><p>Probabilistic CDF calculation, one for each double cdf passed in. See documentation for progQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probCDFS-java.lang.Object-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probCDFS</h4>
|
|
<pre>public static java.lang.Object probCDFS(java.lang.Object colname,
|
|
java.lang.Object cdfs)</pre>
|
|
<div class="block"><p>Probabilistic CDF calculation, one for each double cdf passed in. See documentation for probQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probPMFS-java.lang.Object-java.lang.Object-long-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockList">
|
|
<li class="blockList">
|
|
<h4>probPMFS</h4>
|
|
<pre>public static java.lang.Object probPMFS(java.lang.Object colname,
|
|
java.lang.Object pmfs,
|
|
long k)</pre>
|
|
<div class="block"><p>Returns an approximation to the Probability Mass Function (PMF) of the input stream given a set of splitPoints (values). See <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>. See documentation for probQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
<a name="probPMFS-java.lang.Object-java.lang.Object-">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="blockListLast">
|
|
<li class="blockList">
|
|
<h4>probPMFS</h4>
|
|
<pre>public static java.lang.Object probPMFS(java.lang.Object colname,
|
|
java.lang.Object pmfs)</pre>
|
|
<div class="block"><p>Returns an approximation to the Probability Mass Function (PMF) of the input stream given a set of splitPoints (values). See <a href="https://datasketches.apache.org/api/java/snapshot/apidocs/index.html">DoublesSketch</a>. See documentation for probQuantiles.</p></div>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<!-- ========= END OF CLASS DATA ========= -->
|
|
<!-- ======= START OF BOTTOM NAVBAR ====== -->
|
|
<div class="bottomNav"><a name="navbar.bottom">
|
|
<!-- -->
|
|
</a>
|
|
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
|
|
<a name="navbar.bottom.firstrow">
|
|
<!-- -->
|
|
</a>
|
|
<ul class="navList" title="Navigation">
|
|
<li><a href="../../../overview-summary.html">Overview</a></li>
|
|
<li><a href="package-summary.html">Package</a></li>
|
|
<li class="navBarCell1Rev">Class</li>
|
|
<li><a href="package-tree.html">Tree</a></li>
|
|
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
|
|
<li><a href="../../../index-all.html">Index</a></li>
|
|
<li><a href="../../../help-doc.html">Help</a></li>
|
|
</ul>
|
|
<div class="aboutLanguage"><script type="text/javascript" src="../../../highlight.pack.js"></script>
|
|
<script type="text/javascript"><!--
|
|
hljs.initHighlightingOnLoad();
|
|
//--></script></div>
|
|
</div>
|
|
<div class="subNav">
|
|
<ul class="navList">
|
|
<li><a href="../../../tech/v3/dataset/Modelling.html" title="class in tech.v3.dataset"><span class="typeNameLink">Prev Class</span></a></li>
|
|
<li><a href="../../../tech/v3/dataset/Rolling.html" title="class in tech.v3.dataset"><span class="typeNameLink">Next Class</span></a></li>
|
|
</ul>
|
|
<ul class="navList">
|
|
<li><a href="../../../index.html?tech/v3/dataset/Reductions.html" target="_top">Frames</a></li>
|
|
<li><a href="Reductions.html" target="_top">No Frames</a></li>
|
|
</ul>
|
|
<ul class="navList" id="allclasses_navbar_bottom">
|
|
<li><a href="../../../allclasses-noframe.html">All Classes</a></li>
|
|
</ul>
|
|
<div>
|
|
<script type="text/javascript"><!--
|
|
allClassesLink = document.getElementById("allclasses_navbar_bottom");
|
|
if(window==top) {
|
|
allClassesLink.style.display = "block";
|
|
}
|
|
else {
|
|
allClassesLink.style.display = "none";
|
|
}
|
|
//-->
|
|
</script>
|
|
</div>
|
|
<div>
|
|
<ul class="subNavList">
|
|
<li>Summary: </li>
|
|
<li>Nested | </li>
|
|
<li>Field | </li>
|
|
<li>Constr | </li>
|
|
<li><a href="#method.summary">Method</a></li>
|
|
</ul>
|
|
<ul class="subNavList">
|
|
<li>Detail: </li>
|
|
<li>Field | </li>
|
|
<li>Constr | </li>
|
|
<li><a href="#method.detail">Method</a></li>
|
|
</ul>
|
|
</div>
|
|
<a name="skip.navbar.bottom">
|
|
<!-- -->
|
|
</a></div>
|
|
<!-- ======== END OF BOTTOM NAVBAR ======= -->
|
|
</body>
|
|
</html>
|