Struct datafusion::execution::context::ExecutionConfig

pub struct ExecutionConfig {
    pub target_partitions: usize,
    pub physical_optimizers: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>,
    pub repartition_joins: bool,
    pub repartition_aggregations: bool,
    pub repartition_windows: bool,
    pub runtime: RuntimeConfig,
    /* private fields */
}

Configuration options for execution context

Fields

target_partitions: usize

Number of partitions for query execution. Increasing partitions can increase concurrency.

physical_optimizers: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>

Responsible for optimizing a physical execution plan

repartition_joins: bool

Should DataFusion repartition data using the join keys to execute joins in parallel using the provided target_partitions level

repartition_aggregations: bool

Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided target_partitions level

repartition_windows: bool

Should DataFusion repartition data using the partition keys to execute window functions in parallel using the provided target_partitions level

runtime: RuntimeConfig

Runtime configurations such as memory threshold and local disk for spill

Implementations

impl ExecutionConfig

pub fn new() -> Self

Create an execution config with default setting

pub fn with_target_partitions(self, n: usize) -> Self

Customize target_partitions

pub fn with_batch_size(self, n: usize) -> Self

Customize batch size

pub fn with_query_planner(
    self,
    query_planner: Arc<dyn QueryPlanner + Send + Sync>
) -> Self

Replace the default query planner

pub fn with_optimizer_rules(
    self,
    optimizers: Vec<Arc<dyn OptimizerRule + Send + Sync>>
) -> Self

Replace the optimizer rules

pub fn with_physical_optimizer_rules(
    self,
    physical_optimizers: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>
) -> Self

Replace the physical optimizer rules

pub fn add_optimizer_rule(
    self,
    optimizer_rule: Arc<dyn OptimizerRule + Send + Sync>
) -> Self

Adds a new OptimizerRule

pub fn add_physical_optimizer_rule(
    self,
    optimizer_rule: Arc<dyn PhysicalOptimizerRule + Send + Sync>
) -> Self

Adds a new PhysicalOptimizerRule

pub fn with_default_catalog_and_schema(
    self,
    catalog: impl Into<String>,
    schema: impl Into<String>
) -> Self

Selects a name for the default catalog and schema

pub fn create_default_catalog_and_schema(self, create: bool) -> Self

Controls whether the default catalog and schema will be automatically created

pub fn with_information_schema(self, enabled: bool) -> Self

Enables or disables the inclusion of information_schema virtual tables

pub fn with_repartition_joins(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for joins to improve parallelism

pub fn with_repartition_aggregations(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for aggregations to improve parallelism

pub fn with_repartition_windows(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for window functions to improve parallelism

pub fn with_parquet_pruning(self, enabled: bool) -> Self

Enables or disables the use of pruning predicate for parquet readers to skip row groups

pub fn with_runtime_config(self, config: RuntimeConfig) -> Self

Customize runtime config

pub fn with_existing_memory_manager(self, existing: Arc<MemoryManager>) -> Self

Use an an existing MemoryManager

pub fn with_memory_limit(
    self,
    max_memory: usize,
    memory_fraction: f64
) -> Result<Self>

Specify the total memory to use while running the DataFusion plan to max_memory * memory_fraction in bytes.

Note DataFusion does not yet respect this limit in all cases.

pub fn with_existing_disk_manager(self, existing: Arc<DiskManager>) -> Self

Use an an existing DiskManager

pub fn with_temp_file_path(self, path: impl Into<PathBuf>) -> Self

Use the specified path to create any needed temporary files

Trait Implementations

impl Clone for ExecutionConfig

fn clone(&self) -> ExecutionConfig

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for ExecutionConfig

fn default() -> Self

Returns the “default value” for a type.