feat(workflow-operator): vectorize every named text column, not just one

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala

@@ -33,6 +33,7 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc {
     pyb"""$getImportStatements
        |from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
        |from sklearn.pipeline import make_pipeline
+       |from sklearn.compose import ColumnTransformer
        |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
        |import numpy as np
        |from pytexera import *
@@ -41,10 +42,11 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc {
        |    def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
        |        Y = table[$target]
        |        X = table.drop($target, axis=1)
-       |        X = ${if (countVectorizer) pyb"X[$text]" else "X"}
        |        if port == 0:
-       |            self.model = make_pipeline(${if (countVectorizer) "CountVectorizer(),"
-    else ""} ${if (tfidfTransformer) "TfidfTransformer()," else ""} ${getImportStatements
+       |            self.model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if (
+      tfidfTransformer
+    ) "TfidfTransformer(),"
+    else ""} ${getImportStatements
       .split(" ")
       .last}()).fit(X, Y)
        |        else:

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala

@@ -19,7 +19,12 @@
 
 package org.apache.texera.amber.operator.sklearn
 
-import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyDescription}
+import com.fasterxml.jackson.annotation.{
+  JsonFormat,
+  JsonIgnore,
+  JsonProperty,
+  JsonPropertyDescription
+}
 import com.kjetland.jackson.jsonSchema.annotations.{
   JsonSchemaInject,
   JsonSchemaInt,
@@ -50,12 +55,15 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor {
   var countVectorizer: Boolean = false
 
   @JsonSchemaTitle("Text Attribute")
-  @JsonPropertyDescription("Attribute in your dataset with text to vectorize.")
+  @JsonPropertyDescription("Attributes in your dataset with text to vectorize.")
+  // A workflow written before this field took several columns holds a bare string
+  // here, which reads as a list of one.
+  @JsonFormat(`with` = Array(JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY))
   @JsonSchemaInject(
     strings = Array(
       new JsonSchemaString(
         path = CommonOpDescAnnotation.autofill,
-        value = CommonOpDescAnnotation.attributeName
+        value = CommonOpDescAnnotation.attributeNameList
       ),
       new JsonSchemaString(path = HideAnnotation.hideTarget, value = "countVectorizer"),
       new JsonSchemaString(path = HideAnnotation.hideType, value = HideAnnotation.Type.equals),
@@ -65,7 +73,7 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor {
       new JsonSchemaInt(path = CommonOpDescAnnotation.autofillAttributeOnPort, value = 0)
     )
   )
-  var text: EncodableString = _
+  var text: List[EncodableString] = List()
 
   @JsonSchemaTitle("Tfidf Transformer")
   @JsonPropertyDescription("Transform a count matrix to a normalized tf or tf-idf representation.")
@@ -79,6 +87,30 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor {
   )
   var tfidfTransformer: Boolean = false
 
+  /** The pipeline step that turns the named text columns into features, empty when
+    * Count Vectorizer is off.
+    *
+    * One `CountVectorizer` per column rather than one over all of them:
+    * `CountVectorizer` takes a flat sequence of documents, so handing it several
+    * columns reads them as one document each rather than as the rows. A
+    * `ColumnTransformer` gives each its own and concatenates the results, keeping
+    * the column a word came from distinguishable through the feature's prefix.
+    *
+    * The steps are named by position, not after the column, so that a column whose
+    * name carries a double underscore cannot collide with the separator
+    * `get_feature_names_out` puts between step and feature.
+    *
+    * `renderColumn` writes one column name in the caller's dialect: an encoded
+    * expression for the operator, a literal for the standalone script.
+    */
+  @JsonIgnore
+  protected def vectorizerStage(renderColumn: EncodableString => String): String =
+    if (!countVectorizer) ""
+    else
+      text.zipWithIndex
+        .map { case (column, i) => s"""("text$i", CountVectorizer(), ${renderColumn(column)})""" }
+        .mkString("ColumnTransformer([", ", ", "]),")
+
   @JsonIgnore
   def getImportStatements: String
 

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala

@@ -33,6 +33,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc {
   override def generatePythonCode(): String =
     pyb"""$getImportStatements
        |from sklearn.pipeline import make_pipeline
+       |from sklearn.compose import ColumnTransformer
        |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
        |import numpy as np
        |from pytexera import *
@@ -41,8 +42,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc {
        |    def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
        |        Y = table[$target]
        |        X = table.drop($target, axis=1)
-       |        X = ${if (countVectorizer) pyb"X[$text]" else "X"}
-       |        model = make_pipeline(${if (countVectorizer) "CountVectorizer()," else ""} ${if (
+       |        model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if (
       tfidfTransformer
     ) "TfidfTransformer(),"
     else ""} ${getImportStatements.split(" ").last}()).fit(X, Y)

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnAdaptiveBoostingOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnBaggingOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnBaggingOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnBernoulliNaiveBayesOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala

@@ -56,7 +56,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers {
   ): SklearnKNNOpDesc = {
     val d = new SklearnKNNOpDesc
     d.target = "label"
-    d.text = "docs"
+    d.text = List("docs")
     d.countVectorizer = countVectorizer
     d.tfidfTransformer = tfidfTransformer
     d
@@ -69,7 +69,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers {
     code should include(s"Y = table[${decodeExpr("label")}]")
     code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)")
     // Feature-column path: X is kept whole, the text attribute is never read.
-    code should include("X = X\n")
+    code should not include "ColumnTransformer("
     code should not include decodeExpr("docs")
     normalized(code) should include(
       "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)"
@@ -81,27 +81,45 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers {
 
   it should "select the text column and prepend CountVectorizer when countVectorizer is on" in {
     val code = descriptor(countVectorizer = true).generatePythonCode()
-    code should include(s"X = X[${decodeExpr("docs")}]")
-    code should not include "X = X\n"
+    // ColumnTransformer selects the columns itself, so X stays the whole frame.
     normalized(code) should include(
-      "self.model = make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)"
+      s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr(
+        "docs"
+      )})]), KNeighborsClassifier()).fit(X, Y)"""
     )
     code should not include "TfidfTransformer()"
   }
 
+  // One CountVectorizer per column: it reads a flat sequence of documents, so
+  // several columns handed to one would be read as a document each. The steps are
+  // named by position, keeping a column named with a double underscore away from
+  // the separator get_feature_names_out uses.
+  it should "give each named column its own CountVectorizer" in {
+    val d = descriptor(countVectorizer = true)
+    d.text = List("title", "body")
+    normalized(d.generatePythonCode()) should include(
+      s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr(
+        "title"
+      )}), ("text1", CountVectorizer(), ${decodeExpr(
+        "body"
+      )})]), KNeighborsClassifier()).fit(X, Y)"""
+    )
+  }
+
   it should "chain CountVectorizer before TfidfTransformer when both flags are on" in {
     val code =
       descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode()
-    code should include(s"X = X[${decodeExpr("docs")}]")
     normalized(code) should include(
-      "self.model = make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"
+      s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr(
+        "docs"
+      )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"""
     )
   }
 
   it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in {
     val code = descriptor(tfidfTransformer = true).generatePythonCode()
     // Without countVectorizer there is no text-column selection.
-    code should include("X = X\n")
+    code should not include "ColumnTransformer("
     code should not include decodeExpr("docs")
     normalized(code) should include(
       "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)"

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnComplementNaiveBayesOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnDecisionTreeOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnDummyClassifierOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnExtraTreeOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnExtraTreeOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnExtraTreesOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnExtraTreesOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnGradientBoostingOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnGradientBoostingOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnKNNOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnKNNOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnLinearSVMOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnLinearSVMOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnLogisticRegressionCVOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnLogisticRegressionOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnLogisticRegressionOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnMultiLayerPerceptronOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnMultinomialNaiveBayesOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnNearestCentroidOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnPassiveAggressiveOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnPerceptronOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnPerceptronOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnProbabilityCalibrationOpDescSpec extends AnyFlatSpec with Matchers
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnProbabilityCalibrationOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnRandomForestOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnRandomForestOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnRidgeCVOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnRidgeCVOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnRidgeOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnRidgeOpDesc.getOutputSchemas" should

common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala

@@ -44,7 +44,7 @@ class SklearnSDGOpDescSpec extends AnyFlatSpec with Matchers {
     d.countVectorizer shouldBe false
     d.tfidfTransformer shouldBe false
     d.target shouldBe null
-    d.text shouldBe null
+    d.text shouldBe empty
   }
 
   "SklearnSDGOpDesc.getOutputSchemas" should