def build_spark_session(app_name, memory='10g', executors=16): return SparkSession.builder\ .appName(app_name)\ .config("spark.driver.maxResultSize", memory)\ .config('spark.driver.memory', memory)\ .config('spark.executor.memory', memory)\ .config("spark.dynamicAllocation.enabled", 'true')\ .config("spark.dynamicAllocation.minExecutors", 4)\ .config("spark.dynamicAllocation.maxExecutors", 24)\ .config("spark.dynamicAllocation.initialExecutors", 4)\ .getOrCreate()