/
spark.template.conf
19 lines (19 loc) · 1.5 KB
/
spark.template.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
spark.kubernetes.container.image ${APP_IMAGE}
spark.eventLog.dir ${S3A_LOG_PATH}
spark.hadoop.fs.s3a.endpoint s3.${AWS_REGION}.amazonaws.com
spark.kubernetes.driverEnv.S3A_DATA_PATH ${S3A_DATA_PATH}
spark.executorEnv.S3A_DATA_PATH ${S3A_DATA_PATH}
spark.kubernetes.authenticate.driver.serviceAccountName spark
spark.kubernetes.container.image.pullPolicy Always
spark.executor.instances 2
spark.executor.cores 1
spark.task.maxFailures 3
spark.executor.memory 1g
spark.driver.memory 1g
spark.kubernetes.driver.secretKeyRef.AWS_ACCESS_KEY_ID aws-secrets:aws-access-key
spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY aws-secrets:aws-secret-key
spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID aws-secrets:aws-access-key
spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY aws-secrets:aws-secret-key
spark.executor.extraJavaOptions -Dcom.amazonaws.services.s3.enableV4=true
spark.driver.extraJavaOptions -Dcom.amazonaws.services.s3.enableV4=true
spark.eventLog.enabled true