Struct datafusion::execution::context::SessionConfig

source · [−]

pub struct SessionConfig {
    pub target_partitions: usize,
    pub repartition_joins: bool,
    pub repartition_aggregations: bool,
    pub repartition_windows: bool,
    pub parquet_pruning: bool,
    pub config_options: ConfigOptions,
    /* private fields */
}

Expand description

Configuration options for session context

Fields

target_partitions: usize

Number of partitions for query execution. Increasing partitions can increase concurrency.

repartition_joins: bool

Should DataFusion repartition data using the join keys to execute joins in parallel using the provided target_partitions level

repartition_aggregations: bool

Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided target_partitions level

repartition_windows: bool

Should DataFusion repartition data using the partition keys to execute window functions in parallel using the provided target_partitions level

parquet_pruning: bool

Should DataFusion parquet reader using the predicate to prune data

config_options: ConfigOptions

Configuration options

Implementations

source

impl SessionConfig

source

pub fn new() -> Self

Create an execution config with default setting

source

pub fn from_env() -> Self

Create an execution config with config options read from the environment

source

pub fn set(self, key: &str, value: ScalarValue) -> Self

Set a configuration option

source

pub fn set_bool(self, key: &str, value: bool) -> Self

Set a boolean configuration option

source

pub fn set_u64(self, key: &str, value: u64) -> Self

Set a generic u64 configuration option

source

pub fn with_batch_size(self, n: usize) -> Self

Customize batch size

source

pub fn with_target_partitions(self, n: usize) -> Self

Customize target_partitions

source

pub fn with_default_catalog_and_schema(
 self,
 catalog: impl Into<String>,
 schema: impl Into<String>
) -> Self

Selects a name for the default catalog and schema

source

pub fn create_default_catalog_and_schema(self, create: bool) -> Self

Controls whether the default catalog and schema will be automatically created

source

pub fn with_information_schema(self, enabled: bool) -> Self

Enables or disables the inclusion of information_schema virtual tables

source

pub fn with_repartition_joins(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for joins to improve parallelism

source

pub fn with_repartition_aggregations(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for aggregations to improve parallelism

source

pub fn with_repartition_windows(self, enabled: bool) -> Self

Enables or disables the use of repartitioning for window functions to improve parallelism

source

pub fn with_parquet_pruning(self, enabled: bool) -> Self

Enables or disables the use of pruning predicate for parquet readers to skip row groups

source

pub fn batch_size(&self) -> usize

Get the currently configured batch size

source

pub fn config_options(&self) -> &ConfigOptions

Get the current configuration options

source

pub fn to_props(&self) -> HashMap<String, String>

Convert configuration options to name-value pairs with values converted to strings. Note that this method will eventually be deprecated and replaced by [config_options].

source

pub fn with_extension<T>(self, ext: Arc<T>) -> Self where
T: Send + Sync + 'static,

Add extensions.

Extensions can be used to attach extra data to the session config – e.g. tracing information or caches. Extensions are opaque and the types are unknown to DataFusion itself, which makes them extremely flexible. ¹

Extensions are stored within an Arc so they do NOT require Clone. The are immutable. If you need to modify their state over their lifetime – e.g. for caches – you need to establish some for of interior mutability.

Extensions are indexed by their type T. If multiple values of the same type are provided, only the last one will be kept.

You may use get_extension to retrieve extensions.

Example

use std::sync::Arc;
use datafusion::execution::context::SessionConfig;

// application-specific extension types
struct Ext1(u8);
struct Ext2(u8);
struct Ext3(u8);

let ext1a = Arc::new(Ext1(10));
let ext1b = Arc::new(Ext1(11));
let ext2 = Arc::new(Ext2(2));

let cfg = SessionConfig::default()
    // will only remember the last Ext1
    .with_extension(Arc::clone(&ext1a))
    .with_extension(Arc::clone(&ext1b))
    .with_extension(Arc::clone(&ext2));

let ext1_received = cfg.get_extension::<Ext1>().unwrap();
assert!(!Arc::ptr_eq(&ext1_received, &ext1a));
assert!(Arc::ptr_eq(&ext1_received, &ext1b));

let ext2_received = cfg.get_extension::<Ext2>().unwrap();
assert!(Arc::ptr_eq(&ext2_received, &ext2));

assert!(cfg.get_extension::<Ext3>().is_none());