init research
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
|
||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||
|
||||
plugins {
|
||||
application
|
||||
kotlin("jvm")
|
||||
|
||||
id("org.jetbrains.kotlinx.dataframe")
|
||||
|
||||
// only mandatory if `kotlin.dataframe.add.ksp=false` in gradle.properties
|
||||
id("com.google.devtools.ksp")
|
||||
}
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
mavenLocal() // in case of local dataframe development
|
||||
}
|
||||
|
||||
application.mainClass = "org.jetbrains.kotlinx.dataframe.examples.youtube.YoutubeKt"
|
||||
|
||||
dependencies {
|
||||
// implementation("org.jetbrains.kotlinx:dataframe:X.Y.Z")
|
||||
implementation(project(":"))
|
||||
implementation(libs.kotlin.datetimeJvm)
|
||||
}
|
||||
|
||||
tasks.withType<KotlinCompile> {
|
||||
compilerOptions.jvmTarget = JvmTarget.JVM_1_8
|
||||
}
|
||||
|
||||
kotlin {
|
||||
compilerOptions {
|
||||
jvmTarget = JvmTarget.JVM_1_8
|
||||
freeCompilerArgs.add("-Xjdk-release=8")
|
||||
}
|
||||
}
|
||||
|
||||
tasks.withType<JavaCompile> {
|
||||
sourceCompatibility = JavaVersion.VERSION_1_8.toString()
|
||||
targetCompatibility = JavaVersion.VERSION_1_8.toString()
|
||||
options.release.set(8)
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package org.jetbrains.kotlinx.dataframe.examples.youtube
|
||||
|
||||
val apiKey: String = TODO("Insert your API key here")
|
||||
const val basePath = "https://www.googleapis.com/youtube/v3"
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
@file:ImportDataSchema(
|
||||
"SearchResponse",
|
||||
"src/main/resources/searchResponse.json",
|
||||
)
|
||||
@file:ImportDataSchema(
|
||||
"StatisticsResponse",
|
||||
"src/main/resources/statisticsResponse.json",
|
||||
)
|
||||
|
||||
package org.jetbrains.kotlinx.dataframe.examples.youtube
|
||||
|
||||
import kotlinx.datetime.Instant
|
||||
import org.jetbrains.kotlinx.dataframe.AnyFrame
|
||||
import org.jetbrains.kotlinx.dataframe.AnyRow
|
||||
import org.jetbrains.kotlinx.dataframe.DataRow
|
||||
import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
|
||||
import org.jetbrains.kotlinx.dataframe.api.*
|
||||
import org.jetbrains.kotlinx.dataframe.dataTypes.IFRAME
|
||||
import org.jetbrains.kotlinx.dataframe.dataTypes.IMG
|
||||
import org.jetbrains.kotlinx.dataframe.io.read
|
||||
import java.net.URL
|
||||
|
||||
fun load(path: String) = DataRow.read("$basePath/$path&key=$apiKey")
|
||||
|
||||
fun load(path: String, maxPages: Int): AnyFrame {
|
||||
val rows = mutableListOf<AnyRow>()
|
||||
var pagePath = path
|
||||
do {
|
||||
val row = load(pagePath)
|
||||
rows.add(row)
|
||||
val next = row.getValueOrNull<String>("nextPageToken")
|
||||
pagePath = "$path&pageToken=$next"
|
||||
|
||||
} while (next != null && rows.size < maxPages)
|
||||
return rows.concat()
|
||||
}
|
||||
|
||||
fun main() {
|
||||
val searchRequest = "cute%20cats"
|
||||
val resultsPerPage = 50
|
||||
val maxPages = 5
|
||||
|
||||
val videoId by column<String>("id")
|
||||
val channel by columnGroup()
|
||||
|
||||
val videos = load("search?q=$searchRequest&maxResults=$resultsPerPage&part=snippet", maxPages)
|
||||
.convertTo<SearchResponse> {
|
||||
convert<String?>().with { it.toString() }
|
||||
convert<Int?>().with { it ?: 0 }
|
||||
}
|
||||
.items.concat()
|
||||
.dropNulls { id.videoId }
|
||||
.select { id.videoId into videoId and snippet }
|
||||
.distinct()
|
||||
.parse()
|
||||
.convert { colsAtAnyDepth().colsOf<URL>() }.with {
|
||||
IMG(it, maxHeight = 150)
|
||||
}.add("video") {
|
||||
val id = videoId()
|
||||
IFRAME("http://www.youtube.com/embed/$id")
|
||||
}.move { snippet.title and snippet.publishTime }.toTop()
|
||||
.move { snippet.channelId and snippet.channelTitle }.under(channel)
|
||||
.remove { snippet }
|
||||
|
||||
val stats = videos[videoId]
|
||||
.chunked(50)
|
||||
.map {
|
||||
val ids = it.joinToString("%2C")
|
||||
load("videos?part=statistics&id=$ids").cast<StatisticsResponse>()
|
||||
}.asColumnGroup()
|
||||
.items.concat()
|
||||
.select { id and statistics.allCols() }
|
||||
.parse()
|
||||
|
||||
val withStat = videos.join(stats) { videoId match right.id }
|
||||
|
||||
val viewCount by column<Int>()
|
||||
val publishTime by column<Instant>()
|
||||
|
||||
val channels = withStat
|
||||
.groupBy { channel }.sum { viewCount }
|
||||
.sortByDesc { viewCount }
|
||||
.flatten()
|
||||
|
||||
channels.print(borders = true, columnTypes = true)
|
||||
|
||||
val growth = withStat
|
||||
.select { publishTime and viewCount }
|
||||
.convert { publishTime and viewCount }.toLong()
|
||||
.sortBy { publishTime }
|
||||
.cumSum { viewCount }
|
||||
|
||||
growth.print(borders = true, columnTypes = true)
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
{
|
||||
"kind": "youtube#searchListResponse",
|
||||
"etag": "nl77cg-yrK-TW2q2RtoGXrdkkfo",
|
||||
"nextPageToken": "CAUQAA",
|
||||
"regionCode": "NL",
|
||||
"pageInfo": {
|
||||
"totalResults": 1000000,
|
||||
"resultsPerPage": 5
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"kind": "youtube#searchResult",
|
||||
"etag": "gsRtDXx5RZlp-qILhP65o2oF-go",
|
||||
"id": {
|
||||
"kind": "youtube#video",
|
||||
"videoId": "Dix58mO0Pbc"
|
||||
},
|
||||
"snippet": {
|
||||
"publishedAt": "2022-08-10T14:30:04Z",
|
||||
"channelId": "UC7wafFu5c8AO0YF5U7R7xFA",
|
||||
"title": "Cat TV for Cats to Watch 😺 Summer birds and ducks by the lake 🐦 Cute squirrels 🐿 8 Hours(4K HDR)",
|
||||
"description": "8 hours of pleasing video for cats, dogs, parrots, or other nature lovers to enjoy. It can relax your kitten or puppy and minimize ...",
|
||||
"thumbnails": {
|
||||
"default": {
|
||||
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/default.jpg",
|
||||
"width": 120,
|
||||
"height": 90
|
||||
},
|
||||
"medium": {
|
||||
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/mqdefault.jpg",
|
||||
"width": 320,
|
||||
"height": 180
|
||||
},
|
||||
"high": {
|
||||
"url": "https://i.ytimg.com/vi/Dix58mO0Pbc/hqdefault.jpg",
|
||||
"width": 480,
|
||||
"height": 360
|
||||
}
|
||||
},
|
||||
"channelTitle": "Birder King",
|
||||
"liveBroadcastContent": "none",
|
||||
"publishTime": "2022-08-10T14:30:04Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "youtube#searchResult",
|
||||
"etag": "_7QEwCZHKtgnPTcYmsxNaol-I0Q",
|
||||
"id": {
|
||||
"kind": "youtube#video",
|
||||
"videoId": "bGsN7jzp5DE"
|
||||
},
|
||||
"snippet": {
|
||||
"publishedAt": "2022-08-09T17:00:30Z",
|
||||
"channelId": "UCINb0wqPz-A0dV9nARjJlOQ",
|
||||
"title": "Cat Is Obsessed With His Tiny Love Bird | The Dodo Odd Couples",
|
||||
"description": "This cat is glued to his favorite little love bird and even climbs inside her cage to hang out longer This video is dedicated to ...",
|
||||
"thumbnails": {
|
||||
"default": {
|
||||
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/default.jpg",
|
||||
"width": 120,
|
||||
"height": 90
|
||||
},
|
||||
"medium": {
|
||||
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/mqdefault.jpg",
|
||||
"width": 320,
|
||||
"height": 180
|
||||
},
|
||||
"high": {
|
||||
"url": "https://i.ytimg.com/vi/bGsN7jzp5DE/hqdefault.jpg",
|
||||
"width": 480,
|
||||
"height": 360
|
||||
}
|
||||
},
|
||||
"channelTitle": "The Dodo",
|
||||
"liveBroadcastContent": "none",
|
||||
"publishTime": "2022-08-09T17:00:30Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "youtube#searchResult",
|
||||
"etag": "IHNyBgppiApI3KGzkUV5AuPMftM",
|
||||
"id": {
|
||||
"kind": "youtube#video",
|
||||
"videoId": "U1OxDRxNEMM"
|
||||
},
|
||||
"snippet": {
|
||||
"publishedAt": "2022-08-10T14:45:00Z",
|
||||
"channelId": "UCcnThqTwvub5ykbII9WkR5g",
|
||||
"title": "Funny animals - Funny cats / dogs - Funny animal videos 218",
|
||||
"description": "Funny animals! Compilation number 218. Only the best! Sit back and charge positively Funny animal videos (funny cats, dogs ...",
|
||||
"thumbnails": {
|
||||
"default": {
|
||||
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/default.jpg",
|
||||
"width": 120,
|
||||
"height": 90
|
||||
},
|
||||
"medium": {
|
||||
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/mqdefault.jpg",
|
||||
"width": 320,
|
||||
"height": 180
|
||||
},
|
||||
"high": {
|
||||
"url": "https://i.ytimg.com/vi/U1OxDRxNEMM/hqdefault.jpg",
|
||||
"width": 480,
|
||||
"height": 360
|
||||
}
|
||||
},
|
||||
"channelTitle": "Happy Dog",
|
||||
"liveBroadcastContent": "none",
|
||||
"publishTime": "2022-08-10T14:45:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "youtube#searchResult",
|
||||
"etag": "0OYjMrtwAzJH7yo_jOPvF2hwSao",
|
||||
"id": {
|
||||
"kind": "youtube#video",
|
||||
"videoId": "ByH9LuSILxU"
|
||||
},
|
||||
"snippet": {
|
||||
"publishedAt": "2020-06-19T02:18:53Z",
|
||||
"channelId": "UC8hC-augAnujJeprhjI0YkA",
|
||||
"title": "Baby Cats - Cute and Funny Cat Videos Compilation #34 | Aww Animals",
|
||||
"description": "Baby cats are amazing creature because they are the cutest and most funny. Watching funny baby cats is the hardest try not to ...",
|
||||
"thumbnails": {
|
||||
"default": {
|
||||
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/default.jpg",
|
||||
"width": 120,
|
||||
"height": 90
|
||||
},
|
||||
"medium": {
|
||||
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/mqdefault.jpg",
|
||||
"width": 320,
|
||||
"height": 180
|
||||
},
|
||||
"high": {
|
||||
"url": "https://i.ytimg.com/vi/ByH9LuSILxU/hqdefault.jpg",
|
||||
"width": 480,
|
||||
"height": 360
|
||||
}
|
||||
},
|
||||
"channelTitle": "Aww Animals",
|
||||
"liveBroadcastContent": "none",
|
||||
"publishTime": "2020-06-19T02:18:53Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "youtube#searchResult",
|
||||
"etag": "S1UukOVi_sofJQLHSU0jX5GSv2M",
|
||||
"id": {
|
||||
"kind": "youtube#video",
|
||||
"videoId": "VkqVsCPAIag"
|
||||
},
|
||||
"snippet": {
|
||||
"publishedAt": "2022-08-10T11:03:15Z",
|
||||
"channelId": "UCHBnS9TR-4h2nvuiiq3XCAA",
|
||||
"title": "Awesome SO Cute Cat ! Cute and Funny Cat Videos to Keep You Smiling! 🐱",
|
||||
"description": "The featured clips in our video are used with permission from the original video owners. The highlight clips can be done by our ...",
|
||||
"thumbnails": {
|
||||
"default": {
|
||||
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/default.jpg",
|
||||
"width": 120,
|
||||
"height": 90
|
||||
},
|
||||
"medium": {
|
||||
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/mqdefault.jpg",
|
||||
"width": 320,
|
||||
"height": 180
|
||||
},
|
||||
"high": {
|
||||
"url": "https://i.ytimg.com/vi/VkqVsCPAIag/hqdefault.jpg",
|
||||
"width": 480,
|
||||
"height": 360
|
||||
}
|
||||
},
|
||||
"channelTitle": "Best awesome",
|
||||
"liveBroadcastContent": "none",
|
||||
"publishTime": "2022-08-10T11:03:15Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"kind": "youtube#videoListResponse",
|
||||
"etag": "rHk7psLWXLIjjx8rGeiNKUxrD-s",
|
||||
"items": [
|
||||
{
|
||||
"kind": "youtube#video",
|
||||
"etag": "hiKGiry1Gc19FmHigb3sMfjnzP8",
|
||||
"id": "uHKfrz65KSU",
|
||||
"statistics": {
|
||||
"viewCount": "67715094",
|
||||
"likeCount": "641192",
|
||||
"favoriteCount": "0",
|
||||
"commentCount": "22174"
|
||||
}
|
||||
}
|
||||
],
|
||||
"pageInfo": {
|
||||
"totalResults": 1,
|
||||
"resultsPerPage": 1
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user