ajet/df-research
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
df-research/tech.ml.dataset/docs/javadoc/tech/v3/libs/Parquet.html
ajet bdf064f54d init research
2026-02-08 11:20:43 -10:00

317 lines
12 KiB
HTML
Vendored
Raw Permalink Blame History

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_312) on Sun Dec 11 09:30:03 MST 2022 -->
<title>Parquet (tech.ml.dataset Documentation)</title>
<meta name="date" content="2022-12-11">
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="Parquet (tech.ml.dataset Documentation)";
}
}
catch(err) {
}
//-->
var methods = {"i0":9,"i1":9,"i2":9,"i3":9,"i4":9};
var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
<div class="aboutLanguage"><script async src="https://www.googletagmanager.com/gtag/js?id=G-RGTB4J7LGP"</script><script>window.dataLayer = window.dataLayer || [];\nfunction gtag(){dataLayer.push(arguments);}\ngtag('js', new Date());\ngtag('config', 'G-RGTB4J7LGP');</div>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../tech/v3/libs/Arrow.html" title="class in tech.v3.libs"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li>Next&nbsp;Class</li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?tech/v3/libs/Parquet.html" target="_top">Frames</a></li>
<li><a href="Parquet.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">tech.v3.libs</div>
<h2 title="Class Parquet" class="title">Class Parquet</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>tech.v3.libs.Parquet</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<hr>
<br>
<pre>public class <span class="typeNameLabel">Parquet</span>
extends java.lang.Object</pre>
<div class="block"><p>Read/write parquet files. Uses the standard hadoop parquet library. One aspect that may be confusing is that when writing files the parquet system decides when to end the record batch so a single dataset may end up as a single parquet file with many record batches.</p>
<p>Note that in the requiring dependencies I remove slf4j. tmd comes with logback-classic by default which is less featureful but far less of a security disaster than slf4j. If you have a setup that already uses slf4j then you should exclude logback-classic from tmd&rsquo;s dependencies.</p>
<p>You must disable debug logging else the parquet system is unreasonably slow. See logging section of <a href="https://techascent.github.io/tech.ml.dataset/tech.v3.libs.parquet.html">parquet namespace</a>.</p>
<p>Required dependencies:</p>
<pre><code class="clojure">org.apache.parquet/parquet-hadoop {:mvn/version &quot;1.12.0&quot;
:exclusions [org.slf4j/slf4j-log4j12]}
org.apache.hadoop/hadoop-common {:mvn/version &quot;3.3.0&quot;
:exclusions [org.slf4j/slf4j-log4j12]}
;; We literally need this for 1 POJO formatting object.
org.apache.hadoop/hadoop-mapreduce-client-core {:mvn/version &quot;3.3.0&quot;
:exclusions [org.slf4j/slf4j-log4j12]}
</code></pre></div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t1" class="tableTab"><span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>static void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/libs/Parquet.html#datasetSeqToParquet-java.lang.Iterable-java.lang.String-java.lang.Object-">datasetSeqToParquet</a></span>(java.lang.Iterable&nbsp;dsSeq,
java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</code>&nbsp;</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>static void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/libs/Parquet.html#datasetToParquet-java.lang.Object-java.lang.String-java.lang.Object-">datasetToParquet</a></span>(java.lang.Object&nbsp;ds,
java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</code>&nbsp;</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>static java.lang.Iterable</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/libs/Parquet.html#parquetMetadata-java.lang.String-">parquetMetadata</a></span>(java.lang.String&nbsp;path)</code>&nbsp;</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>static java.util.Map</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/libs/Parquet.html#parquetToDataset-java.lang.String-java.lang.Object-">parquetToDataset</a></span>(java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</code>&nbsp;</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>static java.lang.Iterable</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../tech/v3/libs/Parquet.html#parquetToDatasetSeq-java.lang.String-java.lang.Object-">parquetToDatasetSeq</a></span>(java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</code>&nbsp;</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="parquetMetadata-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parquetMetadata</h4>
<pre>public static&nbsp;java.lang.Iterable&nbsp;parquetMetadata(java.lang.String&nbsp;path)</pre>
</li>
</ul>
<a name="parquetToDataset-java.lang.String-java.lang.Object-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parquetToDataset</h4>
<pre>public static&nbsp;java.util.Map&nbsp;parquetToDataset(java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</pre>
</li>
</ul>
<a name="parquetToDatasetSeq-java.lang.String-java.lang.Object-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parquetToDatasetSeq</h4>
<pre>public static&nbsp;java.lang.Iterable&nbsp;parquetToDatasetSeq(java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</pre>
</li>
</ul>
<a name="datasetToParquet-java.lang.Object-java.lang.String-java.lang.Object-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>datasetToParquet</h4>
<pre>public static&nbsp;void&nbsp;datasetToParquet(java.lang.Object&nbsp;ds,
java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</pre>
</li>
</ul>
<a name="datasetSeqToParquet-java.lang.Iterable-java.lang.String-java.lang.Object-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>datasetSeqToParquet</h4>
<pre>public static&nbsp;void&nbsp;datasetSeqToParquet(java.lang.Iterable&nbsp;dsSeq,
java.lang.String&nbsp;path,
java.lang.Object&nbsp;options)</pre>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
<div class="aboutLanguage"><script type="text/javascript" src="../../../highlight.pack.js"></script>
<script type="text/javascript"><!--
hljs.initHighlightingOnLoad();
//--></script></div>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../tech/v3/libs/Arrow.html" title="class in tech.v3.libs"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li>Next&nbsp;Class</li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?tech/v3/libs/Parquet.html" target="_top">Frames</a></li>
<li><a href="Parquet.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink