How to explode all predicted classes in Spark NLP into columns

[maziyarpanahi]

How to explode all predicted classes in ClassifierDL, SentimentDL, and MultiClassifierDL to columns?

[maziyarpanahi]

Answered by @Dirkster99 🙏

Workflow to Unpack Category Array into Seperate Columns

df.select('Id', F.explode(F.arrays_zip("category.result", "category.metadata")).alias("entities")).printSchema()

Output:

root
 |-- Id: string (nullable = true)
 |-- entities: struct (nullable = false)
 |    |-- 0: string (nullable = true)
 |    |-- 1: struct (nullable = true)
 |    |    |-- Class1: string (nullable = true)
 |    |    |-- Class2: string (nullable = true)
 |    |    |-- Class3: string (nullable = true)
 |    |    |-- Class4: string (nullable = true)
 |    |    |-- Class5: string (nullable = true)
 |    |    |-- Class6: string (nullable = true)
 |    |    |-- Class7: string (nullable = true)
 |    |    |-- Class8: string (nullable = true)
 |    |    |-- Class9: string (nullable = true)
 |    |    |-- Class10: string (nullable = true)
 |    |    |-- Class11: string (nullable = true)
 |    |    |-- Class12: string (nullable = true)
 |    |    |-- Class13: string (nullable = true)
 |    |    |-- Class14: string (nullable = true)
 |    |    |-- Class15: string (nullable = true)
 |    |    |-- Class16: string (nullable = true)
 |    |    |-- Class17: string (nullable = true)
 ...

 |    |    |-- Class21: string (nullable = true)
 |    |    |-- Class22: string (nullable = true)
 |    |    |-- Class23: string (nullable = true)
 |    |    |-- sentence: string (nullable = true)

result = df.select('Id', F.explode(F.arrays_zip("category.result", "category.metadata")).alias("entities")) \
           .select('Id', F.expr("entities['0']").alias("class") \
                 , F.expr("entities['1']['Class1']").alias("Class1") \
                 , F.expr("entities['1']['Class2']").alias("Class2")
                 , F.expr("entities['1']['Class3']").alias("Class3")
                 , F.expr("entities['1']['Class4']").alias("Class4")
                 , F.expr("entities['1']['Class5']").alias("Class5")
                 , F.expr("entities['1']['Class6']").alias("Class6")
                 , F.expr("entities['1']['Class7']").alias("Class7")
                 , F.expr("entities['1']['Class8']").alias("Class8")
                 , F.expr("entities['1']['Class9']").alias("Class9")
                 , F.expr("entities['1']['Class10']").alias("Class10")
                 , F.expr("entities['1']['Class11']").alias("Class11")
                 , F.expr("entities['1']['Class12']").alias("Class12")
                 , F.expr("entities['1']['Class13']").alias("Class13")
                 , F.expr("entities['1']['Class14']").alias("Class14")
                 , F.expr("entities['1']['Class15']").alias("Class15")
                 , F.expr("entities['1']['Class16']").alias("Class16")
                 , F.expr("entities['1']['Class17']").alias("Class17")
                 , F.expr("entities['1']['Class18']").alias("Class18")
                 , F.expr("entities['1']['Class19']").alias("Class19")
                 , F.expr("entities['1']['Class20']").alias("Class20")
                 , F.expr("entities['1']['Class21']").alias("Class21")
                 , F.expr("entities['1']['Class22']").alias("Class22")
                 , F.expr("entities['1']['Class23']").alias("Class23")
         )

result.show()

Output:

Id   |class  |Class1 | Class2 | ..., |Class23
------+-------+-------+--------+ ...  +-------
1     | Class1 |  0.99  | ...  | 0.75