k8s-spark-cluster.yaml

apiVersion: v1
kind: Service
metadata:
  name: spark-master
spec:
  selector:
    app: spark-master
  ports:
  - name: web-ui
    protocol: TCP
    port: 8080
    targetPort: 8080
  - name: master
    protocol: TCP
    port: 7077
    targetPort: 7077
  - name: master-rest
    protocol: TCP
    port: 6066
    targetPort: 6066
  clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
  name: spark-client
spec:
  selector:
    app: spark-client
  clusterIP: None
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: spark-master
  labels:
    app: spark-master
spec:
  selector:
    matchLabels:
      app: spark-master
  template:
    metadata:
      labels:
        app: spark-master
    spec:
      containers:
      - name: spark-master
        image: bde2020/spark-master:3.3.0-hadoop3.3
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
        - containerPort: 7077
        - containerPort: 6066
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: spark-worker
  labels:
    app: spark-worker
spec:
  selector:
    matchLabels:
      name: spark-worker
  template:
    metadata:
      labels:
        name: spark-worker
    spec:
      containers:
      - name: spark-worker
        image: bde2020/spark-worker:3.3.0-hadoop3.3
        imagePullPolicy: Always
        ports:
        - containerPort: 8081