Go语言解析Spark默认配置字符串
import (
"strings"
)
func parseSparkDefaults(s string) SparkDefault {
lines := strings.Split(s, "\n")
config := SparkDefault{}
for _, line := range lines {
parts := strings.Split(line, "=")
if len(parts) == 2 {
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
switch key {
case "spark.eventLog.dir":
config.OSSDir = value
case "spark.hadoop.fs.s3a.access.key":
config.AccessKey = value
case "spark.hadoop.fs.s3a.secret.key":
config.SecretKey = value
case "spark.hadoop.fs.s3a.endpoint":
config.EndPoint = value
case "spark.hadoop.fs.s3a.connection.ssl.enabled":
config.SSLEnabled = value
case "spark.shuffle.service.enabled":
config.LocalShuffleEnabled = value
case "spark.sql.skewJoin.enabled":
config.SkewJoinEnabled = value
case "spark.sql.adaptive.enabled":
config.AdaptiveEnabled = value
case "spark.sql.optimizer.joinReorder.enabled":
config.JoinReorderEnabled = value
case "spark.kubernetes.driver.podTemplateFile":
config.DriverPodTplPath = value
case "spark.kubernetes.executor.podTemplateFile":
config.ExecutorPodTplPath = value
case "spark.kubernetes.authenticate.driver.serviceAccountName":
config.ServiceAccountName = value
case "spark.kubernetes.container.image":
config.Image = value
}
}
}
return config
}
// Spark默认配置
type SparkDefault struct {
OSSDir string `json:"ossDir"`
AccessKey string `json:"accessKey"`
SecretKey string `json:"secretKey"`
EndPoint string `json:"endpoint"`
SSLEnabled string `json:"sslEnabled"`
LocalShuffleEnabled string `json:"localShuffleEnabled"`
SkewJoinEnabled string `json:"skewJoinEnabled"`
AdaptiveEnabled string `json:"adaptiveEnabled"`
JoinReorderEnabled string `json:"joinReorderEnabled"`
DriverPodTplPath string `json:"driverPodTplPath"`
ExecutorPodTplPath string `json:"executorPodTplPath"`
ServiceAccountName string `json:"serviceAccountName"`
Image string `json:"image"`
}
原文地址: https://www.cveoy.top/t/topic/pgn6 著作权归作者所有。请勿转载和采集!