init research

2026-02-08 11:20:43 -10:00
commit bdf064f54d
3041 changed files with 1592200 additions and 0 deletions
@@ -0,0 +1,12 @@
+## :plugins:expressions-converter
+
+This Kotlin Compiler plugin, used by [:core](../../core), can extract intermediate
+DataFrame expressions from `@TransformDataFrameExpressions` annotated functions.
+
+It is used to generate sample "explainer dataframe" HTML files that can be used as iFrames on the documentation website.
+
+Annotated functions in [core/.../test/.../samples/api](../../core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api)
+are tested, generated, and copied over to [docs/StardustDocs/resources/snippets](../../docs/StardustDocs/resources/snippets) by
+our "explainer" [plugin callback proxy](../../core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/explainer),
+which hooks into [the TestBase class](../../core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/TestBase.kt) and
+retrieves the intermediate DataFrame expressions thanks to this module.
@@ -0,0 +1,70 @@
+plugins {
+    with(convention.plugins) {
+        alias(kotlinJvm8)
+    }
+    with(libs.plugins) {
+        alias(shadow)
+        alias(publisher)
+    }
+}
+
+group = "org.jetbrains.kotlinx.dataframe"
+
+dependencies {
+    compileOnly(libs.kotlin.compiler)
+
+    testImplementation(libs.kotlin.compiler)
+    testImplementation(libs.kotlin.compiler.internal.test.framework)
+
+    testRuntimeOnly(projects.core)
+
+    testRuntimeOnly(libs.kotlin.test)
+    testRuntimeOnly(libs.kotlin.script.runtime)
+    testRuntimeOnly(libs.kotlin.annotations.jvm)
+
+    testImplementation(platform(libs.junit.bom))
+    testImplementation(libs.junit.jupiter)
+    testImplementation(libs.junit.platform.commons)
+    testImplementation(libs.junit.platform.launcher)
+    testImplementation(libs.junit.platform.runner)
+    testImplementation(libs.junit.platform.suite.api)
+
+    testRuntimeOnly(libs.junit.jupiter.engine)
+}
+
+tasks.test {
+    useJUnitPlatform()
+    doFirst {
+        setLibraryProperty("org.jetbrains.kotlin.test.kotlin-stdlib", "kotlin-stdlib")
+        setLibraryProperty("org.jetbrains.kotlin.test.kotlin-reflect", "kotlin-reflect")
+        setLibraryProperty("org.jetbrains.kotlin.test.kotlin-test", "kotlin-test")
+        setLibraryProperty("org.jetbrains.kotlin.test.kotlin-script-runtime", "kotlin-script-runtime")
+        setLibraryProperty("org.jetbrains.kotlin.test.kotlin-annotations-jvm", "kotlin-annotations-jvm")
+    }
+}
+
+sourceSets {
+    main {
+        java.setSrcDirs(listOf("src"))
+        resources.setSrcDirs(listOf("resources"))
+    }
+    test {
+        java.setSrcDirs(listOf("tests", "tests-gen"))
+        resources.setSrcDirs(listOf("testResources"))
+    }
+}
+
+tasks.register<JavaExec>("generateTests") {
+    classpath = sourceSets.test.get().runtimeClasspath
+    mainClass = "org.jetbrains.kotlinx.dataframe.GenerateTestsKt"
+}
+
+fun Test.setLibraryProperty(propName: String, jarName: String) {
+    val path = project.configurations
+        .testRuntimeClasspath.get()
+        .files
+        .find { """$jarName-\d.*jar""".toRegex().matches(it.name) }
+        ?.absolutePath
+        ?: return
+    systemProperty(propName, path)
+}
@@ -0,0 +1,6 @@
+#
+# Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+# Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+#
+
+org.jetbrains.kotlinx.dataframe.ExplainerComponentRegistrar
@@ -0,0 +1,18 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar
+import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
+import org.jetbrains.kotlin.config.CompilerConfiguration
+
+@OptIn(ExperimentalCompilerApi::class)
+public class ExplainerComponentRegistrar : CompilerPluginRegistrar() {
+    override val supportsK2: Boolean
+        get() = true
+
+    override val pluginId: String = "org.jetbrains.kotlinx.dataframe.ExplainerComponentRegistrar"
+
+    override fun ExtensionStorage.registerExtensions(configuration: CompilerConfiguration) {
+        IrGenerationExtension.registerExtension(ExplainerIrGenerationExtension())
+    }
+}
@@ -0,0 +1,14 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+
+public class ExplainerIrGenerationExtension : IrGenerationExtension {
+    override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) {
+        val lowering = ExplainerIrTransformer(pluginContext)
+        for (file in moduleFragment.files) {
+            lowering.lower(file)
+        }
+    }
+}
@@ -0,0 +1,330 @@
+@file:Suppress("ktlint:standard:no-unused-imports")
+
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.backend.common.FileLoweringPass
+import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
+import org.jetbrains.kotlin.descriptors.Modality
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.IrElementBase
+import org.jetbrains.kotlin.ir.IrStatement
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.declarations.IrField
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.declarations.IrParameterKind
+import org.jetbrains.kotlin.ir.declarations.path
+import org.jetbrains.kotlin.ir.expressions.IrBlockBody
+import org.jetbrains.kotlin.ir.expressions.IrBody
+import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.expressions.IrDeclarationReference
+import org.jetbrains.kotlin.ir.expressions.IrExpression
+import org.jetbrains.kotlin.ir.expressions.IrExpressionBody
+import org.jetbrains.kotlin.ir.expressions.IrGetValue
+import org.jetbrains.kotlin.ir.expressions.IrStatementOrigin
+import org.jetbrains.kotlin.ir.expressions.impl.IrCallImplWithShape
+import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrFunctionExpressionImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrGetObjectValueImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrGetValueImpl
+import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI
+import org.jetbrains.kotlin.ir.symbols.impl.IrSimpleFunctionSymbolImpl
+import org.jetbrains.kotlin.ir.symbols.impl.IrValueParameterSymbolImpl
+import org.jetbrains.kotlin.ir.types.classFqName
+import org.jetbrains.kotlin.ir.types.makeNullable
+import org.jetbrains.kotlin.ir.types.typeWith
+import org.jetbrains.kotlin.ir.util.SetDeclarationsParentVisitor
+import org.jetbrains.kotlin.ir.util.defaultType
+import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable
+import org.jetbrains.kotlin.ir.util.isLocal
+import org.jetbrains.kotlin.ir.visitors.IrElementTransformerVoid
+import org.jetbrains.kotlin.ir.visitors.IrTransformer
+import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
+import org.jetbrains.kotlin.name.CallableId
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import java.io.File
+
+public data class ContainingDeclarations(val clazz: IrClass?, val function: IrFunction?, val statementIndex: Int = 0)
+
+@OptIn(UnsafeDuringIrConstructionAPI::class)
+public class ExplainerIrTransformer(public val pluginContext: IrPluginContext) :
+    IrTransformer<ContainingDeclarations>(),
+    FileLoweringPass {
+    public lateinit var file: IrFile
+    public lateinit var source: String
+
+    override fun lower(irFile: IrFile) {
+        var file: File
+        file = File("testData/box/${irFile.path}")
+        if (!file.exists()) {
+            file = File(irFile.path)
+        }
+        this.file = irFile
+        source = file.readText()
+        irFile.transformChildren(this, ContainingDeclarations(null, null))
+        irFile.transformChildrenVoid(object : IrElementTransformerVoid() {
+            override fun visitFunction(declaration: IrFunction): IrStatement {
+                declaration.acceptChildren(SetDeclarationsParentVisitor, declaration)
+                return super.visitFunction(declaration)
+            }
+        })
+    }
+
+    override fun visitBlockBody(body: IrBlockBody, data: ContainingDeclarations): IrBody {
+        for (i in 0 until body.statements.size) {
+            @Suppress("UNCHECKED_CAST")
+            body.statements.set(
+                index = i,
+                element = (body.statements[i] as IrElementBase)
+                    .transform(
+                        transformer = this,
+                        data = data.copy(statementIndex = i),
+                    ) as IrStatement,
+            )
+        }
+        return body
+    }
+
+    override fun visitClass(declaration: IrClass, data: ContainingDeclarations): IrStatement =
+        super.visitClass(declaration, data.copy(clazz = declaration))
+
+    override fun visitFunction(declaration: IrFunction, data: ContainingDeclarations): IrStatement {
+        val annotated = declaration.annotations.any {
+            it.type.classFqName
+                ?.shortName()
+                ?.identifierOrNullIfSpecial
+                ?.equals("TransformDataFrameExpressions") == true
+        }
+        return if (annotated) {
+            super.visitFunction(declaration, data.copy(function = declaration))
+        } else {
+            declaration
+        }
+    }
+
+    override fun visitElement(element: IrElement, data: ContainingDeclarations): IrElement {
+        element.transformChildren(this, data)
+        return element
+    }
+
+    override fun visitField(declaration: IrField, data: ContainingDeclarations): IrStatement {
+        if (declaration.isLocal) {
+            declaration.transformChildren(this, data)
+        }
+        return declaration
+    }
+
+    override fun visitExpressionBody(body: IrExpressionBody, data: ContainingDeclarations): IrBody = body
+
+    public val dataFrameLike: Set<FqName> = setOf(
+        FqName("org.jetbrains.kotlinx.dataframe.api.Pivot"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.ReducedPivot"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.PivotGroupBy"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.ReducedPivotGroupBy"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.SplitWithTransform"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.Merge"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.Split"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.Gather"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.Update"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.Convert"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.FormattedFrame"),
+        FqName("org.jetbrains.kotlinx.dataframe.api.GroupBy"),
+        FqName("org.jetbrains.kotlinx.dataframe.DataFrame"),
+        FqName("org.jetbrains.kotlinx.dataframe.DataRow"),
+    )
+
+    public val explainerPackage: FqName = FqName("org.jetbrains.kotlinx.dataframe.explainer")
+
+    override fun visitGetValue(expression: IrGetValue, data: ContainingDeclarations): IrExpression {
+        if (expression.startOffset < 0) return expression
+        if (expression.type.classFqName in dataFrameLike) {
+            return transformDataFrameExpression(expression, expression.symbol.owner.name, receiver = null, data)
+        }
+        return super.visitExpression(expression, data)
+    }
+
+    // also, what if expression type is not DataFrame, but Unit? and receiver expression is DataFrame at some point
+    override fun visitCall(expression: IrCall, data: ContainingDeclarations): IrExpression {
+        if (expression.startOffset < 0) return expression
+        if (expression.type.classFqName in dataFrameLike) {
+            if (expression.symbol.owner.name == Name.identifier("component1")) return expression
+            val extensionReceiverIndex =
+                expression.symbol.owner.parameters.indexOfFirst { it.kind == IrParameterKind.ExtensionReceiver }
+            var receiver: IrExpression?
+            // expression.arguments[extensionReceiverIndex] = extension callables,
+            // expression.dispatchReceiver = member callables such as "GroupBy.aggregate"
+            if (extensionReceiverIndex >= 0) {
+                receiver = expression.arguments[extensionReceiverIndex]!!
+                val transformedExtensionReceiver = receiver.transform(this, data)
+                expression.arguments[extensionReceiverIndex] = transformedExtensionReceiver
+            } else {
+                receiver = expression.dispatchReceiver
+                val transformedExtensionReceiver = expression.dispatchReceiver?.transform(this, data)
+                expression.dispatchReceiver = transformedExtensionReceiver
+            }
+
+            return transformDataFrameExpression(expression, expression.symbol.owner.name, receiver = receiver, data)
+        }
+        return super.visitExpression(expression, data)
+    }
+
+    private fun transformDataFrameExpression(
+        expression: IrDeclarationReference,
+        ownerName: Name,
+        receiver: IrExpression?,
+        data: ContainingDeclarations,
+    ): IrCall {
+        val alsoReference = pluginContext
+            .referenceFunctions(
+                CallableId(FqName("kotlin"), Name.identifier("also")),
+            ).single()
+
+        val result = IrCallImplWithShape(
+            startOffset = -1,
+            endOffset = -1,
+            type = expression.type,
+            symbol = alsoReference,
+            typeArgumentsCount = 1,
+            valueArgumentsCount = 1,
+            contextParameterCount = 0,
+            hasDispatchReceiver = true,
+            hasExtensionReceiver = true,
+        ).apply {
+            val extensionReceiverIndex =
+                this.symbol.owner.parameters.indexOfFirst { it.kind == IrParameterKind.ExtensionReceiver }
+            if (extensionReceiverIndex >= 0) {
+                this.arguments[extensionReceiverIndex] = expression
+            } else {
+                this.insertExtensionReceiver(expression)
+            }
+
+            typeArguments[0] = expression.type
+
+            val symbol = IrSimpleFunctionSymbolImpl()
+            val alsoLambda = pluginContext.irFactory
+                .createSimpleFunction(
+                    startOffset = -1,
+                    endOffset = -1,
+                    origin = IrDeclarationOrigin.LOCAL_FUNCTION_FOR_LAMBDA,
+                    symbol = symbol,
+                    name = Name.special("<anonymous>"),
+                    visibility = DescriptorVisibilities.LOCAL,
+                    modality = Modality.FINAL,
+                    returnType = pluginContext.irBuiltIns.unitType,
+                    isInline = false,
+                    isExternal = false,
+                    isTailrec = false,
+                    isSuspend = false,
+                    isOperator = false,
+                    isInfix = false,
+                    isExpect = false,
+                ).apply {
+                    // replace all regular value parameters with a single one `it`
+                    parameters = parameters.filterNot { it.kind == IrParameterKind.Regular } +
+                        pluginContext.irFactory.createValueParameter(
+                            startOffset = -1,
+                            endOffset = -1,
+                            origin = IrDeclarationOrigin.DEFINED,
+                            kind = IrParameterKind.Regular,
+                            name = Name.identifier("it"),
+                            type = expression.type,
+                            isAssignable = false,
+                            symbol = IrValueParameterSymbolImpl(),
+                            varargElementType = null,
+                            isCrossinline = false,
+                            isNoinline = false,
+                            isHidden = false,
+                        )
+
+                    val itSymbol = parameters.first { it.kind == IrParameterKind.Regular }.symbol
+                    val source = try {
+                        source.substring(expression.startOffset, expression.endOffset)
+                    } catch (e: Exception) {
+                        throw Exception("$expression ${ownerName.asString()} $source", e)
+                    }
+                    val expressionId = expressionId(expression)
+                    val receiverId = receiver?.let { expressionId(it) }
+                    val valueArguments = buildList<IrExpression?> {
+                        add(source.irConstImpl()) // source: String
+                        add(ownerName.asStringStripSpecialMarkers().irConstImpl()) // name: String
+                        add(IrGetValueImpl(-1, -1, itSymbol)) // df: Any
+                        add(expressionId.irConstImpl()) // id: String
+                        add(receiverId.irConstImpl()) // receiverId: String?
+                        add(data.clazz?.fqNameWhenAvailable?.asString().irConstImpl()) // containingClassFqName: String?
+                        add(data.function?.name?.asString().irConstImpl()) // containingFunName: String?
+                        add(
+                            IrConstImpl.int(
+                                -1,
+                                -1,
+                                pluginContext.irBuiltIns.intType,
+                                data.statementIndex,
+                            ),
+                        ) // statementIndex: Int
+                    }
+                    body = pluginContext.irFactory.createBlockBody(-1, -1).apply {
+                        val callableId = CallableId(
+                            explainerPackage,
+                            FqName("PluginCallbackProxy"),
+                            Name.identifier("doAction"),
+                        )
+                        val doAction = pluginContext.referenceFunctions(callableId).single()
+                        statements += IrCallImplWithShape(
+                            startOffset = -1,
+                            endOffset = -1,
+                            type = doAction.owner.returnType,
+                            symbol = doAction,
+                            typeArgumentsCount = 0,
+                            valueArgumentsCount = valueArguments.size,
+                            contextParameterCount = 0,
+                            hasDispatchReceiver = true,
+                            hasExtensionReceiver = false,
+                        ).apply {
+                            val clazz = ClassId(explainerPackage, Name.identifier("PluginCallbackProxy"))
+                            val plugin = pluginContext.referenceClass(clazz)!!
+                            val pluginType = plugin.owner.defaultType
+                            dispatchReceiver = IrGetObjectValueImpl(-1, -1, pluginType, plugin)
+                            val firstValueArgumentIndex = 1 // skipping dispatch receiver
+                            valueArguments.forEachIndexed { i, argument ->
+                                this.arguments[firstValueArgumentIndex + i] = argument
+                            }
+                        }
+                    }
+                }
+            val alsoLambdaExpression = IrFunctionExpressionImpl(
+                startOffset = -1,
+                endOffset = -1,
+                type = pluginContext.irBuiltIns
+                    .functionN(2)
+                    .typeWith(listOf(expression.type, pluginContext.irBuiltIns.unitType)),
+                function = alsoLambda,
+                origin = IrStatementOrigin.LAMBDA,
+            )
+
+            val firstValueArgumentIndex = this.symbol.owner.parameters
+                .indexOfFirst { it.kind == IrParameterKind.Regular }
+                .takeUnless { it < 0 } ?: this.symbol.owner.parameters.size
+            this.arguments[firstValueArgumentIndex] = alsoLambdaExpression
+        }
+        return result
+    }
+
+    private fun String?.irConstImpl(): IrConstImpl {
+        val nullableString = pluginContext.irBuiltIns.stringType.makeNullable()
+        val argument = if (this == null) {
+            IrConstImpl.constNull(-1, -1, nullableString)
+        } else {
+            IrConstImpl.string(-1, -1, nullableString, this)
+        }
+        return argument
+    }
+
+    private fun expressionId(expression: IrExpression): String {
+        val line = file.fileEntry.getLineNumber(expression.startOffset)
+        val column = file.fileEntry.getColumnNumber(expression.startOffset)
+        return "${file.path}:${line + 1}:${column + 1}"
+    }
+}
@@ -0,0 +1,16 @@
+package org.jetbrains.kotlinx.dataframe.explainer
+
+import org.jetbrains.kotlinx.dataframe.*
+import org.jetbrains.kotlinx.dataframe.api.*
+
+object PluginCallback {
+    fun doAction(any: Any) {}
+}
+
+fun box(): String {
+    val a: GroupBy<*, *> = dataFrameOf("a")(1).groupBy("a")
+    a.also {
+        PluginCallback.doAction(it)
+    }
+    return "OK"
+}
@@ -0,0 +1,25 @@
+package
+
+package org {
+
+    package org.jetbrains {
+
+        package org.jetbrains.kotlinx {
+
+            package org.jetbrains.kotlinx.dataframe {
+
+                package org.jetbrains.kotlinx.dataframe.explainer {
+                    public fun box(): kotlin.String
+
+                    public object PluginCallback {
+                        private constructor PluginCallback()
+                        public final fun doAction(/*0*/ any: kotlin.Any): kotlin.Unit
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,156 @@
+package org.jetbrains.kotlinx.dataframe.explainer
+
+import org.jetbrains.kotlinx.dataframe.*
+import org.jetbrains.kotlinx.dataframe.api.*
+
+@TransformDataFrameExpressions
+fun callChain(df: DataFrame<*>) {
+    val df1 = df
+        .filter { "age"<Int>() > 20 }
+        .groupBy("value")
+        .sum()
+}
+
+interface Person
+
+class Wrapper {
+    val df = dataFrameOf("name", "age", "city", "weight")(
+        "Alice", 15, "London", 54,
+        "Bob", 45, "Dubai", 87,
+        "Charlie", 20, "Moscow", null,
+        "Charlie", 40, "Milan", null,
+        "Bob", 30, "Tokyo", 68,
+        "Alice", 20, null, 55,
+        "Charlie", 30, "Moscow", 90
+    )
+
+    val typed: DataFrame<Person> = df.cast()
+
+    @TransformDataFrameExpressions
+    fun ff() {
+        val name by column<String>()
+        val aggregated = typed.groupBy { name() }.aggregate {
+            (if (name().first().startsWith("A")) first() else null) into "agg"
+        }["agg"]
+    }
+}
+
+@TransformDataFrameExpressions
+fun aggregateDf() {
+    val df = dataFrameOf("firstName", "lastName", "age", "city", "weight", "isHappy")(
+        "Alice", "Cooper", 15, "London", 54, true,
+        "Bob", "Dylan", 45, "Dubai", 87, true,
+        "Charlie", "Daniels", 20, "Moscow", null, false,
+        "Charlie", "Chaplin", 40, "Milan", null, true,
+        "Bob", "Marley", 30, "Tokyo", 68, true,
+        "Alice", "Wolf", 20, null, 55, false,
+        "Charlie", "Byrd", 30, "Moscow", 90, true
+    ).group("firstName", "lastName").into("name")
+
+    df.groupBy("city").aggregate {
+        count() into "total"
+        count { "age"<Int>() > 18 } into "adults"
+        median("age") into "median age"
+        min("age") into "min age"
+        maxBy("age")["name"] into "oldest"
+    }
+}
+
+@TransformDataFrameExpressions
+fun move() {
+    val df = dataFrameOf("firstName", "lastName", "age", "city", "weight", "isHappy")(
+        "Alice", "Cooper", 15, "London", 54, true,
+        "Bob", "Dylan", 45, "Dubai", 87, true,
+        "Charlie", "Daniels", 20, "Moscow", null, false,
+        "Charlie", "Chaplin", 40, "Milan", null, true,
+        "Bob", "Marley", 30, "Tokyo", 68, true,
+        "Alice", "Wolf", 20, null, 55, false,
+        "Charlie", "Byrd", 30, "Moscow", 90, true
+    ).group("firstName", "lastName").into("name")
+
+    df.move("age", "weight").into { pathOf("info", it.name()) }
+}
+
+fun interface PluginCallback {
+    fun doAction(
+        source: String,
+        name: String,
+        df: Any,
+        id: String,
+        receiverId: String?,
+        containingClassFqName: String?,
+        containingFunName: String?,
+        statementIndex: Int
+    )
+}
+
+object PluginCallbackProxy : PluginCallback {
+
+    var action: PluginCallback = PluginCallback { _, _, _, _, _, _, _, _  -> Unit }
+
+    override fun doAction(source: String, name: String, df: Any, id: String, receiverId: String?, containingClassFqName: String?, containingFunName: String?, statemenIndex: Int) {
+        action.doAction(source, name, df, id, receiverId, containingClassFqName, containingFunName, statemenIndex)
+    }
+}
+
+//fun callChainTransformed(df: DataFrame<*>) {
+//    val df1 = df
+//        .filter { "age"<Int>() > 20 }
+//        .also { PluginCallbackProxy.doAction(""".filter { "age"<Int>() > 20 }""", "filter", it) }
+//}
+
+//fun callChainTransformed(df: DataFrame<*>) {
+//    val df1 = df
+//        .filter { "age"<Int>() > 20 }
+//        .also { PluginCallbackProxy.action(""".filter { "age"<Int>() > 20 }""", it) }
+//        .groupBy("something")
+//        .sum()
+//        .also { PluginCallbackProxy.action(""".groupBy("something").sum()""", it) }
+//}
+
+annotation class TransformDataFrameExpressions
+
+fun box(): String {
+    val age by columnOf(10, 21, 30, 1)
+    val value by columnOf("a", "b", "c", "c")
+    val df = dataFrameOf(age, value)
+    val expressions = mutableListOf<String>()
+    PluginCallbackProxy.action = PluginCallback { source, _, df, id, receiverId, containingClassFqName, containingFunName, statementIndex ->
+        println("== Call ==")
+        expressions += source
+        if (df is AnyFrame) {
+            println(source)
+            df.print()
+            println(id)
+            println(receiverId)
+            println(containingClassFqName)
+            println(containingFunName)
+            println("statementIndex = ${statementIndex}")
+        } else {
+            println(df::class)
+        }
+        println("== End ==")
+    }
+
+    println("CallChain")
+    callChain(df)
+    println("expressions = ${expressions}")
+    expressions.clear()
+
+    println("ff")
+    Wrapper().ff()
+    println("expressions = ${expressions}")
+    expressions.clear()
+//    callChainTransformed(df)
+
+    println("aggregateDf")
+    aggregateDf()
+    println("expressions = ${expressions}")
+    expressions.clear()
+
+    println("move")
+    move()
+    println("expressions = ${expressions}")
+    expressions.clear()
+    return "OK"
+}
@@ -0,0 +1,59 @@
+package
+
+package org {
+
+    package org.jetbrains {
+
+        package org.jetbrains.kotlinx {
+
+            package org.jetbrains.kotlinx.dataframe {
+
+                package org.jetbrains.kotlinx.dataframe.explainer {
+                    @org.jetbrains.kotlinx.dataframe.explainer.TransformDataFrameExpressions public fun aggregateDf(): kotlin.Unit
+                    public fun box(): kotlin.String
+                    @org.jetbrains.kotlinx.dataframe.explainer.TransformDataFrameExpressions public fun callChain(/*0*/ df: org.jetbrains.kotlinx.dataframe.DataFrame<*>): kotlin.Unit
+                    @org.jetbrains.kotlinx.dataframe.explainer.TransformDataFrameExpressions public fun move(): kotlin.Unit
+
+                    public interface Person {
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+
+                    public fun interface PluginCallback {
+                        public abstract fun doAction(/*0*/ source: kotlin.String, /*1*/ name: kotlin.String, /*2*/ df: kotlin.Any, /*3*/ id: kotlin.String, /*4*/ receiverId: kotlin.String?, /*5*/ containingClassFqName: kotlin.String?, /*6*/ containingFunName: kotlin.String?, /*7*/ statementIndex: kotlin.Int): kotlin.Unit
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+
+                    public object PluginCallbackProxy : org.jetbrains.kotlinx.dataframe.explainer.PluginCallback {
+                        private constructor PluginCallbackProxy()
+                        public final var action: org.jetbrains.kotlinx.dataframe.explainer.PluginCallback
+                        public open override /*1*/ fun doAction(/*0*/ source: kotlin.String, /*1*/ name: kotlin.String, /*2*/ df: kotlin.Any, /*3*/ id: kotlin.String, /*4*/ receiverId: kotlin.String?, /*5*/ containingClassFqName: kotlin.String?, /*6*/ containingFunName: kotlin.String?, /*7*/ statemenIndex: kotlin.Int): kotlin.Unit
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+
+                    public final annotation class TransformDataFrameExpressions : kotlin.Annotation {
+                        public constructor TransformDataFrameExpressions()
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+
+                    public final class Wrapper {
+                        public constructor Wrapper()
+                        public final val df: org.jetbrains.kotlinx.dataframe.DataFrame<*>
+                        public final val typed: org.jetbrains.kotlinx.dataframe.DataFrame<org.jetbrains.kotlinx.dataframe.explainer.Person>
+                        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+                        @org.jetbrains.kotlinx.dataframe.explainer.TransformDataFrameExpressions public final fun ff(): kotlin.Unit
+                        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+                        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,21 @@
+package org.jetbrains.kotlinx.dataframe.explainer
+
+fun print(any: Any) {}
+
+fun printM(pair: Pair<String, Any>) {
+    val (text, obj) = pair
+    println("# ${text}: $obj")
+}
+
+fun ir() {
+    val df = Any()
+    printM("Any()" to df)
+}
+
+fun box(): String {
+    val df = Any()
+    val df1 = 1 + 2
+    print(df)
+    print(df1)
+    return "OK"
+}
@@ -0,0 +1,20 @@
+package
+
+package org {
+
+    package org.jetbrains {
+
+        package org.jetbrains.kotlinx {
+
+            package org.jetbrains.kotlinx.dataframe {
+
+                package org.jetbrains.kotlinx.dataframe.explainer {
+                    public fun box(): kotlin.String
+                    public fun ir(): kotlin.Unit
+                    public fun print(/*0*/ any: kotlin.Any): kotlin.Unit
+                    public fun printM(/*0*/ pair: kotlin.Pair<kotlin.String, kotlin.Any>): kotlin.Unit
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,40 @@
+
+
+package org.jetbrains.kotlinx.dataframe;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.util.KtTestUtil;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlinx.dataframe.GenerateTestsKt}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("testData/box")
+@TestDataPath("$PROJECT_ROOT")
+public class ExplainerBlackBoxCodegenTestGenerated extends AbstractExplainerBlackBoxCodegenTest {
+  @Test
+  public void testAllFilesPresentInBox() {
+    KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("testData/box"), Pattern.compile("^(.+)\\.kt$"), null, true);
+  }
+
+  @Test
+  @TestMetadata("any.kt")
+  public void testAny() {
+    runTest("testData/box/any.kt");
+  }
+
+  @Test
+  @TestMetadata("df.kt")
+  public void testDf() {
+    runTest("testData/box/df.kt");
+  }
+
+  @Test
+  @TestMetadata("test.kt")
+  public void testTest() {
+    runTest("testData/box/test.kt");
+  }
+}
@@ -0,0 +1,78 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.config.JvmTarget
+import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
+import org.jetbrains.kotlin.test.TargetBackend
+import org.jetbrains.kotlin.test.TestJdkKind
+import org.jetbrains.kotlin.test.backend.BlackBoxCodegenSuppressor
+import org.jetbrains.kotlin.test.backend.handlers.IrPrettyKotlinDumpHandler
+import org.jetbrains.kotlin.test.backend.handlers.IrTextDumpHandler
+import org.jetbrains.kotlin.test.backend.handlers.IrTreeVerifierHandler
+import org.jetbrains.kotlin.test.backend.handlers.JvmBoxRunner
+import org.jetbrains.kotlin.test.backend.ir.JvmIrBackendFacade
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+import org.jetbrains.kotlin.test.builders.classicFrontendHandlersStep
+import org.jetbrains.kotlin.test.builders.irHandlersStep
+import org.jetbrains.kotlin.test.builders.jvmArtifactsHandlersStep
+import org.jetbrains.kotlin.test.builders.psi2IrStep
+import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives
+import org.jetbrains.kotlin.test.frontend.classic.ClassicFrontendFacade
+import org.jetbrains.kotlin.test.frontend.classic.handlers.ClassicDiagnosticsHandler
+import org.jetbrains.kotlin.test.frontend.classic.handlers.DeclarationsDumpHandler
+import org.jetbrains.kotlin.test.model.DependencyKind
+import org.jetbrains.kotlin.test.model.FrontendKinds
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.RuntimeClasspathProvider
+import org.jetbrains.kotlin.test.services.TemporaryDirectoryManager
+import org.jetbrains.kotlin.test.services.TestServices
+import org.jetbrains.kotlin.test.services.configuration.CommonEnvironmentConfigurator
+import org.jetbrains.kotlin.test.services.configuration.JvmEnvironmentConfigurator
+import org.jetbrains.kotlinx.dataframe.services.TemporaryDirectoryManagerImplFixed
+import java.io.File
+
+open class AbstractExplainerBlackBoxCodegenTest : BaseTestRunner() {
+
+    override fun configure(builder: TestConfigurationBuilder): Unit =
+        with(builder) {
+            globalDefaults {
+                frontend = FrontendKinds.ClassicAndFIR
+                targetPlatform = JvmPlatforms.jvm8
+                dependencyKind = DependencyKind.Binary
+                targetBackend = TargetBackend.JVM_IR
+            }
+            defaultDirectives {
+                JvmEnvironmentConfigurationDirectives.JDK_KIND with TestJdkKind.FULL_JDK
+                JvmEnvironmentConfigurationDirectives.JVM_TARGET with JvmTarget.JVM_1_8
+                +JvmEnvironmentConfigurationDirectives.WITH_REFLECT
+            }
+            facadeStep(::ClassicFrontendFacade)
+            commonFirWithPluginFrontendConfiguration()
+            classicFrontendHandlersStep {
+                useHandlers(
+                    ::ClassicDiagnosticsHandler,
+                    ::DeclarationsDumpHandler,
+                )
+            }
+            psi2IrStep()
+            irHandlersStep {
+                useHandlers(
+                    ::IrPrettyKotlinDumpHandler,
+                    ::IrTextDumpHandler,
+                    ::IrTreeVerifierHandler,
+                )
+            }
+            facadeStep(::JvmIrBackendFacade)
+            jvmArtifactsHandlersStep {
+                useHandlers(::JvmBoxRunner)
+            }
+            useConfigurators(::JvmEnvironmentConfigurator, ::CommonEnvironmentConfigurator, ::PluginAnnotationsProvider)
+            useCustomRuntimeClasspathProviders(::MyClasspathProvider)
+            useAfterAnalysisCheckers(::BlackBoxCodegenSuppressor)
+            useAdditionalService<TemporaryDirectoryManager>(::TemporaryDirectoryManagerImplFixed)
+        }
+
+    class MyClasspathProvider(testServices: TestServices) : RuntimeClasspathProvider(testServices) {
+        override fun runtimeClassPaths(module: TestModule): List<File> =
+            (classpathFromClassloader(javaClass.classLoader) ?: error("no classpath"))
+    }
+}
@@ -0,0 +1,20 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.test.initIdeaConfiguration
+import org.jetbrains.kotlin.test.runners.AbstractKotlinCompilerTest
+import org.jetbrains.kotlin.test.services.EnvironmentBasedStandardLibrariesPathProvider
+import org.jetbrains.kotlin.test.services.KotlinStandardLibrariesPathProvider
+import org.junit.jupiter.api.BeforeAll
+
+abstract class BaseTestRunner : AbstractKotlinCompilerTest() {
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun setUp() {
+            initIdeaConfiguration()
+        }
+    }
+
+    override fun createKotlinStandardLibrariesPathProvider(): KotlinStandardLibrariesPathProvider =
+        EnvironmentBasedStandardLibrariesPathProvider
+}
@@ -0,0 +1,19 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar
+import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.EnvironmentConfigurator
+import org.jetbrains.kotlin.test.services.TestServices
+
+class ExtensionRegistrarConfigurator(testServices: TestServices) : EnvironmentConfigurator(testServices) {
+    @OptIn(ExperimentalCompilerApi::class)
+    override fun CompilerPluginRegistrar.ExtensionStorage.registerCompilerExtensions(
+        module: TestModule,
+        configuration: CompilerConfiguration,
+    ) {
+        IrGenerationExtension.registerExtension(ExplainerIrGenerationExtension())
+    }
+}
@@ -0,0 +1,13 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.generators.dsl.junit5.generateTestGroupSuiteWithJUnit5
+
+fun main() {
+    generateTestGroupSuiteWithJUnit5 {
+        testGroup(testDataRoot = "testData", testsRoot = "tests-gen") {
+            testClass<AbstractExplainerBlackBoxCodegenTest> {
+                model("box")
+            }
+        }
+    }
+}
@@ -0,0 +1,13 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.EnvironmentConfigurator
+import org.jetbrains.kotlin.test.services.TestServices
+
+class PluginAnnotationsProvider(testServices: TestServices) : EnvironmentConfigurator(testServices) {
+    override fun configureCompilerConfiguration(configuration: CompilerConfiguration, module: TestModule) {
+        configuration.addJvmClasspathRoots(classpathFromClassloader(javaClass.classLoader) ?: error("no classpath"))
+    }
+}
@@ -0,0 +1,479 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlinx.dataframe
+
+import java.io.File
+import java.io.FileInputStream
+import java.io.FileNotFoundException
+import java.net.JarURLConnection
+import java.net.URL
+import java.net.URLClassLoader
+import java.util.jar.JarInputStream
+import kotlin.reflect.KClass
+
+// Kotlin Compiler dependencies
+internal const val KOTLIN_JAVA_STDLIB_JAR = "kotlin-stdlib.jar"
+internal const val KOTLIN_JAVA_REFLECT_JAR = "kotlin-reflect.jar"
+internal const val KOTLIN_JAVA_SCRIPT_RUNTIME_JAR = "kotlin-script-runtime.jar"
+internal const val TROVE4J_JAR = "trove4j.jar"
+internal const val KOTLIN_SCRIPTING_COMPILER_JAR = "kotlin-scripting-compiler.jar"
+internal const val KOTLIN_SCRIPTING_COMPILER_EMBEDDABLE_JAR = "kotlin-scripting-compiler-embeddable.jar"
+internal const val KOTLIN_SCRIPTING_COMPILER_IMPL_JAR = "kotlin-scripting-compiler-impl.jar"
+internal const val KOTLIN_SCRIPTING_COMPILER_IMPL_EMBEDDABLE_JAR = "kotlin-scripting-compiler-impl-embeddable.jar"
+internal const val KOTLIN_SCRIPTING_COMMON_JAR = "kotlin-scripting-common.jar"
+internal const val KOTLIN_SCRIPTING_JVM_JAR = "kotlin-scripting-jvm.jar"
+
+internal const val KOTLIN_COMPILER_NAME = "kotlin-compiler"
+internal const val KOTLIN_COMPILER_JAR = "$KOTLIN_COMPILER_NAME.jar"
+
+private val JAR_COLLECTIONS_CLASSES_PATHS = arrayOf("BOOT-INF/classes", "WEB-INF/classes")
+private val JAR_COLLECTIONS_LIB_PATHS = arrayOf("BOOT-INF/lib", "WEB-INF/lib")
+private val JAR_COLLECTIONS_KEY_PATHS = JAR_COLLECTIONS_CLASSES_PATHS + JAR_COLLECTIONS_LIB_PATHS
+internal const val JAR_MANIFEST_RESOURCE_NAME = "META-INF/MANIFEST.MF"
+
+internal const val KOTLIN_SCRIPT_CLASSPATH_PROPERTY = "kotlin.script.classpath"
+internal const val KOTLIN_COMPILER_CLASSPATH_PROPERTY = "kotlin.compiler.classpath"
+internal const val KOTLIN_COMPILER_JAR_PROPERTY = "kotlin.compiler.jar"
+internal const val KOTLIN_STDLIB_JAR_PROPERTY = "kotlin.java.stdlib.jar"
+internal const val KOTLIN_REFLECT_JAR_PROPERTY = "kotlin.java.reflect.jar"
+
+// obsolete name, but maybe still used in the wild
+// TODO: consider removing
+internal const val KOTLIN_RUNTIME_JAR_PROPERTY = "kotlin.java.runtime.jar"
+internal const val KOTLIN_SCRIPT_RUNTIME_JAR_PROPERTY = "kotlin.script.runtime.jar"
+
+private val validClasspathFilesExtensions = setOf("jar", "zip", "java")
+private val validJarCollectionFilesExtensions = setOf("jar", "war", "zip")
+
+class ClasspathExtractionException(message: String) : Exception(message)
+
+fun classpathFromClassloader(currentClassLoader: ClassLoader, unpackJarCollections: Boolean = false): List<File>? {
+    val processedJars = hashSetOf<File>()
+    val unpackJarCollectionsDir by lazy {
+        File.createTempFile("unpackedJarCollections", null).canonicalFile.apply {
+            delete()
+            mkdir()
+            setReadable(false, false)
+            setWritable(false, false)
+            setExecutable(false, false)
+            setReadable(true, true)
+            setWritable(true, true)
+            setExecutable(true, true)
+
+            Runtime.getRuntime().addShutdownHook(
+                Thread {
+                    deleteRecursively()
+                },
+            )
+        }
+    }
+    return allRelatedClassLoaders(currentClassLoader).flatMap { classLoader ->
+        var classPath = emptySequence<File>()
+        if (unpackJarCollections &&
+            JAR_COLLECTIONS_KEY_PATHS.any { classLoader.getResource(it)?.file?.isNotEmpty() == true }
+        ) {
+            // if cache dir is specified, find all jar collections (spring boot fat jars and WARs so far, and unpack it accordingly
+            val jarCollections = JAR_COLLECTIONS_KEY_PATHS
+                .asSequence()
+                .flatMap { currentClassLoader.getResources(it).asSequence() }
+                .mapNotNull {
+                    it.toContainingJarOrNull()?.takeIf { file ->
+                        // additionally mark/check processed collection jars since unpacking is expensive
+                        file.extension in validJarCollectionFilesExtensions && processedJars.add(file)
+                    }
+                }
+            classPath +=
+                jarCollections
+                    .flatMap { it.unpackJarCollection(unpackJarCollectionsDir) }
+                    .filter { it.isValidClasspathFile() }
+        }
+        classPath += when (classLoader) {
+            is URLClassLoader -> {
+                classLoader.urLs.asSequence().mapNotNull { url -> url.toValidClasspathFileOrNull() }
+            }
+
+            else -> {
+                classLoader.classPathFromGetUrlsMethodOrNull()
+                    ?: classLoader.classPathFromTypicalResourceUrls()
+            }
+        }
+        classPath
+    }.filter { processedJars.add(it) }
+        .toList()
+        .takeIf { it.isNotEmpty() }
+}
+
+internal fun URL.toValidClasspathFileOrNull(): File? =
+    (toContainingJarOrNull() ?: toFileOrNull())?.takeIf { it.isValidClasspathFile() }
+
+internal fun File.isValidClasspathFile(): Boolean =
+    isDirectory || (isFile && extension in validClasspathFilesExtensions)
+
+private fun ClassLoader.classPathFromGetUrlsMethodOrNull(): Sequence<File>? =
+    try {
+        // e.g. for IDEA platform UrlClassLoader
+        val getUrls = this::class.java.getMethod("getUrls")
+        getUrls.isAccessible = true
+        val result = getUrls.invoke(this) as? List<Any?>
+        result?.asSequence()?.filterIsInstance<URL>()?.mapNotNull { it.toValidClasspathFileOrNull() }
+    } catch (e: Throwable) {
+        null
+    }
+
+internal class ClassLoaderResourceRootFIlePathCalculator(private val keyResourcePath: String) {
+    private var keyResourcePathDepth = -1
+
+    operator fun invoke(resourceFile: File): File {
+        if (keyResourcePathDepth < 0) {
+            keyResourcePathDepth =
+                if (keyResourcePath.isBlank()) {
+                    0
+                } else {
+                    keyResourcePath.trim('/').count { it == '/' } + 1
+                }
+        }
+        var root = resourceFile
+        for (i in 0 until keyResourcePathDepth) {
+            root = root.parentFile
+        }
+        return root
+    }
+}
+
+internal fun ClassLoader.rawClassPathFromKeyResourcePath(keyResourcePath: String): Sequence<File> {
+    val resourceRootCalc = ClassLoaderResourceRootFIlePathCalculator(keyResourcePath)
+    return getResources(keyResourcePath).asSequence().mapNotNull { url ->
+        if (url.protocol == "jar") {
+            (url.openConnection() as? JarURLConnection)?.jarFileURL?.toFileOrNull()
+        } else {
+            url.toFileOrNull()?.let { resourceRootCalc(it) }
+        }
+    }
+}
+
+fun ClassLoader.classPathFromTypicalResourceUrls(): Sequence<File> =
+// roots without manifest cases are detected in some test scenarios
+// manifests without containing directory entries are detected in some optimized jars, e.g. after proguard
+// TODO: investigate whether getting resources with empty name works in all situations
+    (rawClassPathFromKeyResourcePath("") + rawClassPathFromKeyResourcePath(JAR_MANIFEST_RESOURCE_NAME))
+        .distinct()
+        .filter { it.isValidClasspathFile() }
+
+private fun File.unpackJarCollection(rootTempDir: File): Sequence<File> {
+    val targetDir = File.createTempFile(nameWithoutExtension, null, rootTempDir).apply {
+        delete()
+        mkdir()
+    }
+
+    return try {
+        ArrayList<File>().apply {
+            JarInputStream(FileInputStream(this@unpackJarCollection)).use { jarInputStream ->
+                for (classesDir in JAR_COLLECTIONS_CLASSES_PATHS) {
+                    add(File(targetDir, classesDir))
+                }
+                do {
+                    val entry = jarInputStream.nextJarEntry
+                    if (entry != null) {
+                        try {
+                            if (!entry.isDirectory) {
+                                val file = File(targetDir, entry.name)
+                                if (JAR_COLLECTIONS_LIB_PATHS.any { entry.name.startsWith("$it/") }) {
+                                    add(file)
+                                }
+                                file.parentFile.mkdirs()
+                                file.outputStream().use { outputStream ->
+                                    jarInputStream.copyTo(outputStream)
+                                    outputStream.flush()
+                                }
+                            }
+                        } finally {
+                            jarInputStream.closeEntry()
+                        }
+                    }
+                } while (entry != null)
+            }
+        }.asSequence()
+    } catch (e: Throwable) {
+        targetDir.deleteRecursively()
+        throw e
+    }
+}
+
+fun classpathFromClasspathProperty(): List<File>? =
+    System.getProperty("java.class.path")
+        ?.split(String.format("\\%s", File.pathSeparatorChar).toRegex())
+        ?.dropLastWhile(String::isEmpty)
+        ?.map(::File)
+
+fun classpathFromClass(classLoader: ClassLoader, klass: KClass<out Any>): List<File>? =
+    classpathFromFQN(classLoader, klass.qualifiedName!!)
+
+fun classpathFromClass(klass: KClass<out Any>): List<File>? = classpathFromClass(klass.java.classLoader, klass)
+
+inline fun <reified T : Any> classpathFromClass(): List<File>? = classpathFromClass(T::class)
+
+fun classpathFromFQN(classLoader: ClassLoader, fqn: String): List<File>? {
+    val clp = "${fqn.replace('.', '/')}.class"
+    return classLoader
+        .rawClassPathFromKeyResourcePath(clp)
+        .filter { it.isValidClasspathFile() }
+        .toList()
+        .takeIf { it.isNotEmpty() }
+}
+
+fun File.matchMaybeVersionedFile(baseName: String) =
+    name == baseName ||
+        // for classes dirs
+        name == baseName.removeSuffix(".jar") ||
+        Regex(Regex.escape(baseName.removeSuffix(".jar")) + "(-\\d.*)?\\.jar").matches(name)
+
+fun File.hasParentNamed(baseName: String): Boolean =
+    nameWithoutExtension == baseName || parentFile?.hasParentNamed(baseName) ?: false
+
+private const val KOTLIN_COMPILER_EMBEDDABLE_JAR = "$KOTLIN_COMPILER_NAME-embeddable.jar"
+
+// Iterating over classloaders tree in a regular, parent-first order
+private fun allRelatedClassLoaders(
+    clsLoader: ClassLoader,
+    visited: MutableSet<ClassLoader> = HashSet(),
+): Sequence<ClassLoader> {
+    if (!visited.add(clsLoader)) return emptySequence()
+
+    val singleParent = clsLoader.parent
+    if (singleParent != null) {
+        return sequenceOf(singleParent).flatMap { allRelatedClassLoaders(it, visited) } + clsLoader
+    }
+
+    return try {
+        val arrayOfClassLoaders = getParentClassLoaders(clsLoader)
+        // TODO: PluginClassLoader uses filtering (mustBeLoadedByPlatform), consider using the same logic, if possible
+        // (untill proper compiling from classloader instead of classpath is implemented)
+        arrayOfClassLoaders.asSequence().flatMap { allRelatedClassLoaders(it, visited) } + clsLoader
+    } catch (e: Throwable) {
+        sequenceOf(clsLoader)
+    }
+}
+
+private fun getParentClassLoaders(clsLoader: ClassLoader): Array<ClassLoader> =
+    try {
+        getParentsForNewClassLoader(clsLoader)
+    } catch (exception: NoSuchMethodException) {
+        try {
+            getParentsForOldClassLoader(clsLoader)
+        } catch (exception: NoSuchFieldException) {
+            // Possibly idea sources and kotlin compiler had diverged
+            emptyArray()
+        }
+    }
+
+@Throws(NoSuchFieldException::class)
+private fun getParentsForOldClassLoader(clsLoader: ClassLoader): Array<ClassLoader> {
+    // Correct way of getting parents in com.intellij.ide.plugins.cl.PluginClassLoader from IDEA 202 and earlier
+    val field = clsLoader.javaClass.getDeclaredField("myParents") // com.intellij.ide.plugins.cl.PluginClassLoader
+    field.isAccessible = true
+
+    @Suppress("UNCHECKED_CAST")
+    return field.get(clsLoader) as Array<ClassLoader>
+}
+
+@Throws(NoSuchMethodException::class)
+private fun getParentsForNewClassLoader(clsLoader: ClassLoader): Array<ClassLoader> {
+    // Correct way of getting parents in com.intellij.ide.plugins.cl.PluginClassLoader from IDEA 203+
+    val method = clsLoader.javaClass.getDeclaredMethod("getAllParents")
+    method.isAccessible = true
+
+    @Suppress("UNCHECKED_CAST")
+    return method.invoke(clsLoader) as Array<ClassLoader>
+}
+
+internal fun List<File>.takeIfContainsAll(vararg keyNames: String): List<File>? =
+    takeIf { classpath ->
+        keyNames.all { key -> classpath.any { it.matchMaybeVersionedFile(key) } }
+    }
+
+internal fun List<File>.filterIfContainsAll(vararg keyNames: String): List<File>? {
+    val foundKeys = mutableSetOf<String>()
+    val res = arrayListOf<File>()
+    for (cpentry in this) {
+        for (prefix in keyNames) {
+            if (cpentry.matchMaybeVersionedFile(prefix) || (cpentry.isDirectory && cpentry.hasParentNamed(prefix))) {
+                foundKeys.add(prefix)
+                res.add(cpentry)
+                break
+            }
+        }
+    }
+    return res.takeIf { foundKeys.containsAll(keyNames.asList()) }
+}
+
+internal fun List<File>.takeIfContainsAny(vararg keyNames: String): List<File>? =
+    takeIf { classpath ->
+        keyNames.any { key -> classpath.any { it.matchMaybeVersionedFile(key) } }
+    }
+
+fun scriptCompilationClasspathFromContextOrNull(
+    vararg keyNames: String,
+    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+    wholeClasspath: Boolean = false,
+    unpackJarCollections: Boolean = false,
+): List<File>? {
+    fun List<File>.takeAndFilter() =
+        when {
+            isEmpty() -> null
+            wholeClasspath -> takeIfContainsAll(*keyNames)
+            else -> filterIfContainsAll(*keyNames)
+        }
+
+    val fromProperty = System.getProperty(KOTLIN_SCRIPT_CLASSPATH_PROPERTY)?.split(File.pathSeparator)?.map(::File)
+    if (fromProperty != null) return fromProperty
+
+    return classpathFromClassloader(classLoader, unpackJarCollections)?.takeAndFilter()
+        ?: classpathFromClasspathProperty()?.takeAndFilter()
+}
+
+fun scriptCompilationClasspathFromContext(
+    vararg keyNames: String,
+    classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+    wholeClasspath: Boolean = false,
+    unpackJarCollections: Boolean = false,
+): List<File> =
+    scriptCompilationClasspathFromContextOrNull(
+        *keyNames,
+        classLoader = classLoader,
+        wholeClasspath = wholeClasspath,
+        unpackJarCollections = unpackJarCollections,
+    ) ?: throw ClasspathExtractionException(
+        "Unable to get script compilation classpath from context, please specify explicit classpath via \"$KOTLIN_SCRIPT_CLASSPATH_PROPERTY\" property",
+    )
+
+object KotlinJars {
+
+    private val explicitCompilerClasspath: List<File>? by lazy {
+        System.getProperty(KOTLIN_COMPILER_CLASSPATH_PROPERTY)?.split(File.pathSeparator)?.map(::File)
+            ?: System.getProperty(KOTLIN_COMPILER_JAR_PROPERTY)
+                ?.let(::File)
+                ?.takeIf(File::exists)
+                ?.let { listOf(it) }
+    }
+
+    val compilerClasspath: List<File> by lazy {
+        findCompilerClasspath(withScripting = false)
+    }
+
+    val compilerWithScriptingClasspath: List<File> by lazy {
+        findCompilerClasspath(withScripting = true)
+    }
+
+    private fun findCompilerClasspath(withScripting: Boolean): List<File> {
+        val kotlinCompilerJars = listOf(
+            KOTLIN_COMPILER_JAR,
+            KOTLIN_COMPILER_EMBEDDABLE_JAR,
+        )
+        val kotlinLibsJars = listOf(
+            KOTLIN_JAVA_STDLIB_JAR,
+            KOTLIN_JAVA_REFLECT_JAR,
+            KOTLIN_JAVA_SCRIPT_RUNTIME_JAR,
+            TROVE4J_JAR,
+        )
+        val kotlinScriptingJars = if (withScripting) {
+            listOf(
+                KOTLIN_SCRIPTING_COMPILER_JAR,
+                KOTLIN_SCRIPTING_COMPILER_EMBEDDABLE_JAR,
+                KOTLIN_SCRIPTING_COMPILER_IMPL_JAR,
+                KOTLIN_SCRIPTING_COMPILER_IMPL_EMBEDDABLE_JAR,
+                KOTLIN_SCRIPTING_COMMON_JAR,
+                KOTLIN_SCRIPTING_JVM_JAR,
+            )
+        } else {
+            emptyList()
+        }
+
+        val kotlinBaseJars = kotlinCompilerJars + kotlinLibsJars + kotlinScriptingJars
+
+        val classpath = explicitCompilerClasspath
+            // search classpath from context classloader and `java.class.path` property
+            ?: (
+                classpathFromFQN(
+                    classLoader = Thread.currentThread().contextClassLoader,
+                    fqn = "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler",
+                )
+                    ?: classpathFromClassloader(Thread.currentThread().contextClassLoader)?.takeIf { it.isNotEmpty() }
+                    ?: classpathFromClasspathProperty()
+            )?.filter { f -> kotlinBaseJars.any { f.matchMaybeVersionedFile(it) } }
+                ?.takeIf { it.isNotEmpty() }
+        // if autodetected, additionally check for presence of the compiler jars
+        if (classpath == null ||
+            (
+                explicitCompilerClasspath == null &&
+                    classpath.none { f ->
+                        kotlinCompilerJars.any { f.matchMaybeVersionedFile(it) }
+                    }
+            )
+        ) {
+            throw FileNotFoundException(
+                "Cannot find kotlin compiler jar, set kotlin.compiler.classpath property to proper location",
+            )
+        }
+        return classpath
+    }
+
+    fun getLib(
+        propertyName: String,
+        jarName: String,
+        markerClass: KClass<*>,
+        classLoader: ClassLoader? = null,
+    ): File? =
+        getExplicitLib(propertyName, jarName)
+            ?: run {
+                val requestedClassloader = classLoader ?: Thread.currentThread().contextClassLoader
+                val byName =
+                    if (requestedClassloader == markerClass.java.classLoader) {
+                        null
+                    } else {
+                        tryGetResourcePathForClassByName(markerClass.java.name, requestedClassloader)
+                    }
+                byName ?: tryGetResourcePathForClass(markerClass.java)
+            }?.takeIf(File::exists)
+
+    fun getLib(
+        propertyName: String,
+        jarName: String,
+        markerClassName: String,
+        classLoader: ClassLoader? = null,
+    ): File? =
+        getExplicitLib(propertyName, jarName)
+            ?: tryGetResourcePathForClassByName(
+                markerClassName,
+                classLoader ?: Thread.currentThread().contextClassLoader,
+            )?.takeIf(File::exists)
+
+    private fun getExplicitLib(propertyName: String, jarName: String) =
+        System.getProperty(propertyName)?.let(::File)?.takeIf(File::exists)
+            ?: explicitCompilerClasspath?.firstOrNull { it.matchMaybeVersionedFile(jarName) }?.takeIf(File::exists)
+
+    val stdlibOrNull: File? by lazy {
+        System.getProperty(KOTLIN_STDLIB_JAR_PROPERTY)?.let(::File)?.takeIf(File::exists)
+            ?: getLib(
+                KOTLIN_RUNTIME_JAR_PROPERTY,
+                KOTLIN_JAVA_STDLIB_JAR,
+                JvmStatic::class,
+            )
+    }
+
+    val stdlib: File by lazy {
+        stdlibOrNull ?: throw Exception(
+            "Unable to find kotlin stdlib, please specify it explicitly via \"$KOTLIN_STDLIB_JAR_PROPERTY\" property",
+        )
+    }
+
+    val reflectOrNull: File? by lazy {
+        getLib(
+            KOTLIN_REFLECT_JAR_PROPERTY,
+            KOTLIN_JAVA_REFLECT_JAR,
+            "kotlin.reflect.full.KClasses", // using a class that is a part of the kotlin-reflect.jar
+        )
+    }
+}
@@ -0,0 +1,9 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+
+fun TestConfigurationBuilder.commonFirWithPluginFrontendConfiguration() {
+    useConfigurators(
+        ::ExtensionRegistrarConfigurator,
+    )
+}
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlinx.dataframe
+
+import java.io.File
+import java.net.JarURLConnection
+import java.net.URI
+import java.net.URL
+
+// Based on an implementation in com.intellij.openapi.application.PathManager.getResourceRoot
+
+internal fun getResourceRoot(context: Class<*>, path: String): String? {
+    var url: URL? = context.getResource(path)
+    if (url == null) {
+        url = ClassLoader.getSystemResource(path.substring(1))
+    }
+    return if (url != null) extractRoot(url, path) else null
+}
+
+private const val JAR_PROTOCOL = "jar"
+private const val FILE_PROTOCOL = "file"
+private const val JAR_SEPARATOR = "!/"
+private const val SCHEME_SEPARATOR = "://"
+
+private fun extractRoot(resourceURL: URL, resourcePath: String): String? {
+    if (!resourcePath.startsWith('/') || resourcePath.startsWith('\\')) return null
+
+    var resultPath: String? = null
+    val protocol = resourceURL.protocol
+    if (protocol == FILE_PROTOCOL) {
+        val path = resourceURL.toFileOrNull()!!.path
+        val testPath = path.replace('\\', '/')
+        val testResourcePath = resourcePath.replace('\\', '/')
+        if (testPath.endsWith(testResourcePath, ignoreCase = true)) {
+            resultPath = path.substring(0, path.length - resourcePath.length)
+        }
+    } else if (protocol == JAR_PROTOCOL) {
+        val paths = splitJarUrl(resourceURL.file)
+        if (paths?.first != null) {
+            resultPath = File(paths.first).canonicalPath
+        }
+    }
+
+    return resultPath?.trimEnd(File.separatorChar)
+}
+
+private fun splitJarUrl(url: String): Pair<String, String>? {
+    val pivot = url.indexOf(JAR_SEPARATOR).takeIf { it >= 0 } ?: return null
+
+    val resourcePath = url.substring(pivot + 2)
+    var jarPath = url.substring(0, pivot)
+
+    if (jarPath.startsWith(JAR_PROTOCOL + ":")) {
+        jarPath = jarPath.substring(JAR_PROTOCOL.length + 1)
+    }
+
+    if (jarPath.startsWith(FILE_PROTOCOL)) {
+        try {
+            jarPath = URI(jarPath).toURL().toFileOrNull()!!.path.replace('\\', '/')
+        } catch (e: Exception) {
+            jarPath = jarPath.substring(FILE_PROTOCOL.length)
+            if (jarPath.startsWith(SCHEME_SEPARATOR)) {
+                jarPath = jarPath.substring(SCHEME_SEPARATOR.length)
+            } else if (jarPath.startsWith(':')) {
+                jarPath = jarPath.substring(1)
+            }
+        }
+    }
+    return Pair(jarPath, resourcePath)
+}
+
+fun tryGetResourcePathForClass(aClass: Class<*>): File? {
+    val path = "/" + aClass.name.replace('.', '/') + ".class"
+    return getResourceRoot(aClass, path)?.let {
+        File(it).absoluteFile
+    }
+}
+
+fun getResourcePathForClass(aClass: Class<*>): File =
+    tryGetResourcePathForClass(aClass)
+        ?: throw IllegalStateException("Resource for class: ${aClass.name} not found")
+
+fun tryGetResourcePathForClassByName(name: String, classLoader: ClassLoader): File? =
+    try {
+        classLoader.loadClass(name)?.let(::tryGetResourcePathForClass)
+    } catch (_: ClassNotFoundException) {
+        null
+    } catch (_: NoClassDefFoundError) {
+        null
+    }
+
+internal fun URL.toFileOrNull() =
+    try {
+        File(toURI())
+    } catch (e: IllegalArgumentException) {
+        null
+    } catch (e: java.net.URISyntaxException) {
+        null
+    } ?: run {
+        if (protocol != "file") {
+            null
+        } else {
+            File(file)
+        }
+    }
+
+internal fun URL.toContainingJarOrNull(): File? =
+    if (protocol == "jar") {
+        (openConnection() as? JarURLConnection)?.jarFileURL?.toFileOrNull()
+    } else {
+        null
+    }
@@ -0,0 +1,58 @@
+package org.jetbrains.kotlinx.dataframe.services
+
+import org.jetbrains.kotlin.test.services.TemporaryDirectoryManager
+import org.jetbrains.kotlin.test.services.TestServices
+import org.jetbrains.kotlin.test.services.testInfo
+import org.jetbrains.kotlin.test.util.KtTestUtil
+import java.io.File
+import java.nio.file.Paths
+import java.util.Locale
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.deleteRecursively
+
+// Copied from org.jetbrains.kotlin.test.services.impl.TemporaryDirectoryManagerImpl
+// because it uses NioFiles#deleteRecursively and throws method not found as a result.
+class TemporaryDirectoryManagerImplFixed(testServices: TestServices) : TemporaryDirectoryManager(testServices) {
+    private val cache = mutableMapOf<String, File>()
+    private val rootTempDir = lazy {
+        val testInfo = testServices.testInfo
+        val className = testInfo.className
+        val methodName = testInfo.methodName
+        if (!onWindows && className.length + methodName.length < 255) {
+            return@lazy KtTestUtil.tmpDirForTest(className, methodName)
+        }
+
+        // This code will simplify directory name for windows. This is needed because there can occur errors due to long name
+        val lastDot = className.lastIndexOf('.')
+        val simplifiedClassName = className.substring(lastDot + 1).getOnlyUpperCaseSymbols()
+        val simplifiedMethodName = methodName.getOnlyUpperCaseSymbols()
+        KtTestUtil.tmpDirForTest(simplifiedClassName, simplifiedMethodName)
+    }
+
+    override val rootDir: File
+        get() = rootTempDir.value
+
+    override fun getOrCreateTempDirectory(name: String): File =
+        cache.getOrPut(name) {
+            KtTestUtil.tmpDir(rootDir, name)
+        }
+
+    @OptIn(ExperimentalPathApi::class)
+    override fun cleanupTemporaryDirectories() {
+        cache.clear()
+
+        if (rootTempDir.isInitialized()) {
+            Paths.get(rootDir.path).deleteRecursively()
+        }
+    }
+
+    companion object {
+        private val onWindows: Boolean =
+            System.getProperty("os.name").lowercase(Locale.getDefault()).contains("windows")
+
+        private fun String.getOnlyUpperCaseSymbols(): String =
+            this.filter { it.isUpperCase() || it == '$' }
+                .toList()
+                .joinToString(separator = "")
+    }
+}