Struct datafusion::execution::dataframe_impl::DataFrameImpl

pub struct DataFrameImpl { /* private fields */ }

Implementation of DataFrame API

Implementations

impl DataFrameImpl

pub fn new(
    ctx_state: Arc<Mutex<ExecutionContextState>>,
    plan: &LogicalPlan
) -> Self

Create a new Table based on an existing logical plan

Trait Implementations

impl DataFrame for DataFrameImpl

fn select_columns(&self, columns: &[&str]) -> Result<Arc<dyn DataFrame>>

Apply a projection based on a list of column names

fn select(&self, expr_list: Vec<Expr>) -> Result<Arc<dyn DataFrame>>

Create a projection based on arbitrary expressions

fn filter(&self, predicate: Expr) -> Result<Arc<dyn DataFrame>>

Create a filter based on a predicate expression

fn aggregate(
    &self,
    group_expr: Vec<Expr>,
    aggr_expr: Vec<Expr>
) -> Result<Arc<dyn DataFrame>>

Perform an aggregate query

fn limit(&self, n: usize) -> Result<Arc<dyn DataFrame>>

Limit the number of rows

fn sort(&self, expr: Vec<Expr>) -> Result<Arc<dyn DataFrame>>

Sort by specified sorting expressions

fn join(
    &self,
    right: Arc<dyn DataFrame>,
    join_type: JoinType,
    left_cols: &[&str],
    right_cols: &[&str]
) -> Result<Arc<dyn DataFrame>>

Join with another DataFrame

fn to_logical_plan(&self) -> LogicalPlan

Convert to logical plan

fn collect<'life0, 'async_trait>(
    &'life0 self
) -> Pin<Box<dyn Future<Output = Result<Vec<RecordBatch>>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Convert the logical plan represented by this DataFrame into a physical plan and execute it, collecting all resulting batches into memory

fn show<'life0, 'async_trait>(
    &'life0 self
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Print results.

fn show_limit<'life0, 'async_trait>(
    &'life0 self,
    num: usize
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Print results and limit rows.

fn execute_stream<'life0, 'async_trait>(
    &'life0 self
) -> Pin<Box<dyn Future<Output = Result<SendableRecordBatchStream>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Convert the logical plan represented by this DataFrame into a physical plan and execute it, returning a stream over a single partition

fn collect_partitioned<'life0, 'async_trait>(
    &'life0 self
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<RecordBatch>>>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Convert the logical plan represented by this DataFrame into a physical plan and execute it, collecting all resulting batches into memory while maintaining partitioning

fn execute_stream_partitioned<'life0, 'async_trait>(
    &'life0 self
) -> Pin<Box<dyn Future<Output = Result<Vec<SendableRecordBatchStream>>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    Self: 'async_trait, 

Convert the logical plan represented by this DataFrame into a physical plan and execute it, returning a stream for each partition

fn schema(&self) -> &DFSchema

Returns the schema from the logical plan

fn repartition(
    &self,
    partitioning_scheme: Partitioning
) -> Result<Arc<dyn DataFrame>>

Repartition a DataFrame based on a logical partitioning scheme.

fn explain(&self, verbose: bool, analyze: bool) -> Result<Arc<dyn DataFrame>>

Return a DataFrame with the explanation of its plan so far.

fn registry(&self) -> Arc<dyn FunctionRegistry>

Return a FunctionRegistry used to plan udf’s calls

fn union(&self, dataframe: Arc<dyn DataFrame>) -> Result<Arc<dyn DataFrame>>

Calculate the union two DataFrames. The two DataFrames must have exactly the same schema

fn distinct(&self) -> Result<Arc<dyn DataFrame>>

Calculate the union distinct two DataFrames. The two DataFrames must have exactly the same schema

fn intersect(&self, dataframe: Arc<dyn DataFrame>) -> Result<Arc<dyn DataFrame>>

Calculate the intersection of two DataFrames. The two DataFrames must have exactly the same schema

fn except(&self, dataframe: Arc<dyn DataFrame>) -> Result<Arc<dyn DataFrame>>

Calculate the exception of two DataFrames. The two DataFrames must have exactly the same schema

impl TableProvider for DataFrameImpl

fn as_any(&self) -> &dyn Any

Returns the table provider as Any so that it can be downcast to a specific implementation.

fn schema(&self) -> SchemaRef

Get a reference to the schema for this table

fn table_type(&self) -> TableType

Get the type of this table for metadata/catalog purposes.

fn scan<'life0, 'life1, 'life2, 'async_trait>(
    &'life0 self,
    projection: &'life1 Option<Vec<usize>>,
    filters: &'life2 [Expr],
    limit: Option<usize>
) -> Pin<Box<dyn Future<Output = Result<Arc<dyn ExecutionPlan>>> + Send + 'async_trait>> where
    'life0: 'async_trait,
    'life1: 'async_trait,
    'life2: 'async_trait,
    Self: 'async_trait, 

Create an ExecutionPlan that will scan the table. The table provider will be usually responsible of grouping the source data into partitions that can be efficiently parallelized or distributed.

fn supports_filter_pushdown(
    &self,
    _filter: &Expr
) -> Result<TableProviderFilterPushDown>

Tests whether the table provider can make use of a filter expression to optimise data retrieval.