[Bug]: Jaeger spark job failing with elasticsearch 8.12.1

[CarTerr]

What happened?

if use spark without indexPrefix then run fine
if use spark elasticsearch indexPrefix then get error

Perhaps the problem is the incorrect use of the index jaeger-test:jaeger-span-2024-03-20
The index should be "-" instead of ":"

jaeger-test-jaeger-span-2024-03-20

Exception in thread "main" org.elasticsearch.hadoop.rest.EsHadoopInvalidRequest: [HEAD] on [jaeger-test:jaeger-span-2024-03-20] failed; server[https://192.168.0.14:9200] returned [400|Bad Request:]
at org.elasticsearch.hadoop.rest.RestClient.checkResponse(RestClient.java:487)
at org.elasticsearch.hadoop.rest.RestClient.executeNotFoundAllowed(RestClient.java:457)
at org.elasticsearch.hadoop.rest.RestClient.exists(RestClient.java:549)
at org.elasticsearch.hadoop.rest.RestClient.indexExists(RestClient.java:544)
at org.elasticsearch.hadoop.rest.InitializationUtils.checkIndexStatus(InitializationUtils.java:73)
at org.elasticsearch.hadoop.rest.InitializationUtils.validateSettingsForReading(InitializationUtils.java:279)
at org.elasticsearch.hadoop.rest.RestService.findPartitions(RestService.java:221)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions$lzycompute(AbstractEsRDD.scala:78)
at org.elasticsearch.spark.rdd.AbstractEsRDD.esPartitions(AbstractEsRDD.scala:77)
at org.elasticsearch.spark.rdd.AbstractEsRDD.getPartitions(AbstractEsRDD.scala:47)
at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:294)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:290)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:49)
at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:294)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:290)
at org.apache.spark.Partitioner$.$anonfun$defaultPartitioner$4(Partitioner.scala:78)
at org.apache.spark.Partitioner$.$anonfun$defaultPartitioner$4$adapted(Partitioner.scala:78)
at scala.collection.immutable.List.map(List.scala:293)
at org.apache.spark.Partitioner$.defaultPartitioner(Partitioner.scala:78)
at org.apache.spark.rdd.RDD.$anonfun$groupBy$1(RDD.scala:749)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:410)
at org.apache.spark.rdd.RDD.groupBy(RDD.scala:749)
at org.apache.spark.api.java.JavaRDDLike.groupBy(JavaRDDLike.scala:243)
at org.apache.spark.api.java.JavaRDDLike.groupBy$(JavaRDDLike.scala:239)
at org.apache.spark.api.java.AbstractJavaRDDLike.groupBy(JavaRDDLike.scala:45)
at io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJob.run(ElasticsearchDependenciesJob.java:273)
at io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJob.run(ElasticsearchDependenciesJob.java:249)
at io.jaegertracing.spark.dependencies.DependenciesSparkJob.run(DependenciesSparkJob.java:54)
at io.jaegertracing.spark.dependencies.DependenciesSparkJob.main(DependenciesSparkJob.java:40)

Steps to reproduce

run spark image ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:latest

provisionDataStore:
  cassandra: false

tag: "1.55"

storage:
  type: elasticsearch
  elasticsearch:
    scheme: https
    host: 192.168.0.14
    port: 9200
    user: elastic
    password: 123123
    indexPrefix: jaeger-prod
    nodesWanOnly: true

agent:
  enabled: true
  image: jaegertracing/jaeger-agent
  daemonset:
    useHostPort: true
  serviceAccount:
    create: false
  useHostNetwork: true
  dnsPolicy: ClusterFirstWithHostNet

collector:
  enabled: true
  image: jaegertracing/jaeger-collector
  ingress:
    enabled: true
    hosts:
      - host: collector.jaeger-prod
        servicePort: grpc
  serviceAccount:
    create: false
  cmdlineParams:
    es.tls.enabled: "true"
    es.tls.ca: "/tls/ca.crt"
  extraConfigmapMounts:
    - name: jaeger-tls
      mountPath: /tls
      subPath: ""
      configMap: jaeger-tls
      readOnly: true

query:
  agentSidecar:
    enabled: false
  cmdlineParams:
    es.tls.enabled: "true"
    es.tls.ca: "/tls/ca.crt"
  extraConfigmapMounts:
    - name: jaeger-tls
      mountPath: /tls
      subPath: ""
      configMap: jaeger-tls
      readOnly: true
  ingress:
    enabled: true
    annotations:
      kubernetes.io/tls-acme: "true"
      cert-manager.io/cluster-issuer: letsencrypt-prod
    tls:
      - secretName: jaeger-prod
        hosts:
          - jaeger-prod
    hosts:
      - jaeger-prod
  serviceAccount:
    create: false

spark:
  enabled: true
  image: ghcr.io/jaegertracing/spark-dependencies/spark-dependencies
  tag: latest
  extraEnv:
    - name: JAVA_OPTS
      value: "-Djavax.net.ssl.trustStore=/tls/trust.store -Djavax.net.ssl.trustStorePassword=changeit"
  extraConfigmapMounts:
    - name: jaeger-tls
      mountPath: /tls
      subPath: ""
      configMap: jaeger-tls
      readOnly: true
  serviceAccount:
    create: false

Expected behavior

run cronjob

Relevant log output

No response

Screenshot

No response

Additional context

No response

Jaeger backend version

No response

SDK

No response

Pipeline

No response

Stogage backend

No response

Operating system

No response

Deployment model

No response

Deployment configs

No response