diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala index 92aec692a6e..33c5325b0b9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDesc.scala @@ -33,6 +33,7 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc { pyb"""$getImportStatements |from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer |import numpy as np |from pytexera import * @@ -41,10 +42,11 @@ abstract class SklearnClassifierOpDesc extends SklearnModelOpDesc { | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | Y = table[$target] | X = table.drop($target, axis=1) - | X = ${if (countVectorizer) pyb"X[$text]" else "X"} | if port == 0: - | self.model = make_pipeline(${if (countVectorizer) "CountVectorizer()," - else ""} ${if (tfidfTransformer) "TfidfTransformer()," else ""} ${getImportStatements + | self.model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if ( + tfidfTransformer + ) "TfidfTransformer()," + else ""} ${getImportStatements .split(" ") .last}()).fit(X, Y) | else: diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala index 6665477aeac..f7506ffa820 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala @@ -19,7 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyDescription} +import com.fasterxml.jackson.annotation.{ + JsonFormat, + JsonIgnore, + JsonProperty, + JsonPropertyDescription +} import com.kjetland.jackson.jsonSchema.annotations.{ JsonSchemaInject, JsonSchemaInt, @@ -50,12 +55,15 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { var countVectorizer: Boolean = false @JsonSchemaTitle("Text Attribute") - @JsonPropertyDescription("Attribute in your dataset with text to vectorize.") + @JsonPropertyDescription("Attributes in your dataset with text to vectorize.") + // A workflow written before this field took several columns holds a bare string + // here, which reads as a list of one. + @JsonFormat(`with` = Array(JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY)) @JsonSchemaInject( strings = Array( new JsonSchemaString( path = CommonOpDescAnnotation.autofill, - value = CommonOpDescAnnotation.attributeName + value = CommonOpDescAnnotation.attributeNameList ), new JsonSchemaString(path = HideAnnotation.hideTarget, value = "countVectorizer"), new JsonSchemaString(path = HideAnnotation.hideType, value = HideAnnotation.Type.equals), @@ -65,7 +73,7 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { new JsonSchemaInt(path = CommonOpDescAnnotation.autofillAttributeOnPort, value = 0) ) ) - var text: EncodableString = _ + var text: List[EncodableString] = List() @JsonSchemaTitle("Tfidf Transformer") @JsonPropertyDescription("Transform a count matrix to a normalized tf or tf-idf representation.") @@ -79,6 +87,30 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { ) var tfidfTransformer: Boolean = false + /** The pipeline step that turns the named text columns into features, empty when + * Count Vectorizer is off. + * + * One `CountVectorizer` per column rather than one over all of them: + * `CountVectorizer` takes a flat sequence of documents, so handing it several + * columns reads them as one document each rather than as the rows. A + * `ColumnTransformer` gives each its own and concatenates the results, keeping + * the column a word came from distinguishable through the feature's prefix. + * + * The steps are named by position, not after the column, so that a column whose + * name carries a double underscore cannot collide with the separator + * `get_feature_names_out` puts between step and feature. + * + * `renderColumn` writes one column name in the caller's dialect: an encoded + * expression for the operator, a literal for the standalone script. + */ + @JsonIgnore + protected def vectorizerStage(renderColumn: EncodableString => String): String = + if (!countVectorizer) "" + else + text.zipWithIndex + .map { case (column, i) => s"""("text$i", CountVectorizer(), ${renderColumn(column)})""" } + .mkString("ColumnTransformer([", ", ", "]),") + @JsonIgnore def getImportStatements: String diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala index 3f354cc37f5..6d066a5e9b6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDesc.scala @@ -33,6 +33,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc { override def generatePythonCode(): String = pyb"""$getImportStatements |from sklearn.pipeline import make_pipeline + |from sklearn.compose import ColumnTransformer |from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer |import numpy as np |from pytexera import * @@ -41,8 +42,7 @@ class SklearnTrainingOpDesc extends SklearnModelOpDesc { | def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: | Y = table[$target] | X = table.drop($target, axis=1) - | X = ${if (countVectorizer) pyb"X[$text]" else "X"} - | model = make_pipeline(${if (countVectorizer) "CountVectorizer()," else ""} ${if ( + | model = make_pipeline(${vectorizerStage(c => pyb"$c".toString)} ${if ( tfidfTransformer ) "TfidfTransformer()," else ""} ${getImportStatements.split(" ").last}()).fit(X, Y) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala index 52375db8304..084fa014f3c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnAdaptiveBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala index 3a1759754c4..bc11cade128 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnBaggingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnBaggingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala index b34d9b2e1b9..0d5fa28fc49 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnBernoulliNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala index 9e732002eb9..e964fcca346 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnClassifierOpDescCodegenSpec.scala @@ -56,7 +56,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { ): SklearnKNNOpDesc = { val d = new SklearnKNNOpDesc d.target = "label" - d.text = "docs" + d.text = List("docs") d.countVectorizer = countVectorizer d.tfidfTransformer = tfidfTransformer d @@ -69,7 +69,7 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { code should include(s"Y = table[${decodeExpr("label")}]") code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") // Feature-column path: X is kept whole, the text attribute is never read. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "self.model = make_pipeline( KNeighborsClassifier()).fit(X, Y)" @@ -81,27 +81,45 @@ class SklearnClassifierOpDescCodegenSpec extends AnyFlatSpec with Matchers { it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { val code = descriptor(countVectorizer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") - code should not include "X = X\n" + // ColumnTransformer selects the columns itself, so X stays the whole frame. normalized(code) should include( - "self.model = make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), KNeighborsClassifier()).fit(X, Y)""" ) code should not include "TfidfTransformer()" } + // One CountVectorizer per column: it reads a flat sequence of documents, so + // several columns handed to one would be read as a document each. The steps are + // named by position, keeping a column named with a double underscore away from + // the separator get_feature_names_out uses. + it should "give each named column its own CountVectorizer" in { + val d = descriptor(countVectorizer = true) + d.text = List("title", "body") + normalized(d.generatePythonCode()) should include( + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "title" + )}), ("text1", CountVectorizer(), ${decodeExpr( + "body" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + } + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { val code = descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") normalized(code) should include( - "self.model = make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + s"""self.model = make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" ) } it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { val code = descriptor(tfidfTransformer = true).generatePythonCode() // Without countVectorizer there is no text-column selection. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "self.model = make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala index 0d50eb45775..90b67bdb1ed 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnComplementNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala index 44fbf1aeed7..be316adf0f6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnDecisionTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala index 24b391b6bf5..c670069ae79 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnDummyClassifierOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala index 96b9ec470b8..6c2f424b171 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnExtraTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala index 332bee1ee35..33de266f3c7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnExtraTreesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala index 9c25894dc18..f4cd969c2e1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala index 111482e6057..67b9de06c80 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnGradientBoostingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnGradientBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala index 57dcc06be49..256a298af94 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnKNNOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnKNNOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala index d0153ae37fb..f09c1f7ed99 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLinearSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala index 24589453edc..6bdcb8b99aa 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLogisticRegressionCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala index 077ecb9ac43..6f6b0e10b58 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnLogisticRegressionOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnLogisticRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala index 592f5779568..85a54a28828 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnMultiLayerPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala index 3a6b4debfd2..6573e26cd93 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnMultinomialNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala index db61804a79b..ba173413d57 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnNearestCentroidOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala index d184eb43674..684b008cf7e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnPassiveAggressiveOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala index b28a57c5827..02b932792ce 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala index 2eab3102cb1..53a30df5b55 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnProbabilityCalibrationOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnProbabilityCalibrationOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala index 2f19aff0ad3..63c0f536e5a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRandomForestOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRandomForestOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala index c3fe761c15a..05892469d56 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRidgeCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala index 1f24306bb6d..b13437a7d6e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnRidgeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnRidgeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala index 516374b0874..cc114667d12 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnSDGOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnSDGOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala index b1a7e9f3b0b..d88f9c71d85 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala index 5eeeae389d9..dc26343aa8e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingAdaptiveBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala index dc5916b4b2b..3ac4d1b612b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingBaggingOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingBaggingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala index 7bfe9c0905b..14da4504dbf 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matc d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingBernoulliNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala index 1c30b439b26..b79c579beef 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingComplementNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala index 33031a81d7a..f81a8c40952 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingDecisionTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala index 603ee1b5477..fe3e2bb73f4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingDummyClassifierOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala index f43606ed5ab..509055f06b4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingExtraTreeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala index e404a05c0de..b0d9d1e4594 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingExtraTreesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala index 7d193883ccf..045ae4d979d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Match d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala index dc17a8267a3..90ec0d15de0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingGradientBoostingOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingGradientBoostingOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala index b67d26ccb22..383bdec47f8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingKNNOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingKNNOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala index 7bd4a2af522..31ae5860a15 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLinearRegressionOpDescSpec extends AnyFlatSpec with Matcher d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLinearRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala index 2e0cce8265d..2ae012556b9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLinearSVMOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala index b0386a50111..15e11eaa2d8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLogisticRegressionCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala index e5b04f95a36..1e0b63e562c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingLogisticRegressionOpDescSpec extends AnyFlatSpec with Match d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingLogisticRegressionOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala index d23aca58b45..2e8faaba08b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Mat d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingMultiLayerPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala index 6e1bda2cf12..3ca3c3706a1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Ma d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingMultinomialNaiveBayesOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala index 46afa31dd1b..a9f0d8efbe8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingNearestCentroidOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala index 0c00ca271b5..114393d374d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingOpDescCodegenSpec.scala @@ -56,7 +56,7 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { ): SklearnTrainingKNNOpDesc = { val d = new SklearnTrainingKNNOpDesc d.target = "label" - d.text = "docs" + d.text = List("docs") d.countVectorizer = countVectorizer d.tfidfTransformer = tfidfTransformer d @@ -69,7 +69,7 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { code should include(s"Y = table[${decodeExpr("label")}]") code should include(s"X = table.drop(${decodeExpr("label")}, axis=1)") // Feature-column path: X is kept whole, the text attribute is never read. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include("make_pipeline( KNeighborsClassifier()).fit(X, Y)") code should not include "CountVectorizer()" @@ -78,27 +78,45 @@ class SklearnTrainingOpDescCodegenSpec extends AnyFlatSpec with Matchers { it should "select the text column and prepend CountVectorizer when countVectorizer is on" in { val code = descriptor(countVectorizer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") - code should not include "X = X\n" + // ColumnTransformer selects the columns itself, so X stays the whole frame. normalized(code) should include( - "make_pipeline(CountVectorizer(), KNeighborsClassifier()).fit(X, Y)" + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), KNeighborsClassifier()).fit(X, Y)""" ) code should not include "TfidfTransformer()" } + // One CountVectorizer per column: it reads a flat sequence of documents, so + // several columns handed to one would be read as a document each. The steps are + // named by position, keeping a column named with a double underscore away from + // the separator get_feature_names_out uses. + it should "give each named column its own CountVectorizer" in { + val d = descriptor(countVectorizer = true) + d.text = List("title", "body") + normalized(d.generatePythonCode()) should include( + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "title" + )}), ("text1", CountVectorizer(), ${decodeExpr( + "body" + )})]), KNeighborsClassifier()).fit(X, Y)""" + ) + } + it should "chain CountVectorizer before TfidfTransformer when both flags are on" in { val code = descriptor(countVectorizer = true, tfidfTransformer = true).generatePythonCode() - code should include(s"X = X[${decodeExpr("docs")}]") normalized(code) should include( - "make_pipeline(CountVectorizer(), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" + s"""make_pipeline(ColumnTransformer([("text0", CountVectorizer(), ${decodeExpr( + "docs" + )})]), TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)""" ) } it should "prepend only TfidfTransformer and keep all features when tfidfTransformer is on alone" in { val code = descriptor(tfidfTransformer = true).generatePythonCode() // Without countVectorizer there is no text-column selection. - code should include("X = X\n") + code should not include "ColumnTransformer(" code should not include decodeExpr("docs") normalized(code) should include( "make_pipeline( TfidfTransformer(), KNeighborsClassifier()).fit(X, Y)" diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala index 519a85c4164..dbb0efacefc 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matche d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingPassiveAggressiveOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala index 2ce66d8fdf9..f3641f91743 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingPerceptronOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingPerceptronOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala index 21684da7c76..58e6a3c6428 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingProbabilityCalibrationOpDescSpec extends AnyFlatSpec with M d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingProbabilityCalibrationOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala index 045bbd2b5a7..e41ff84488c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRandomForestOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRandomForestOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala index 2649692c02d..8dfed45ee92 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRidgeCVOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala index 6e2923e4ce5..343463e9c0b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingRidgeOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingRidgeOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala index 26388758431..2c7b0707241 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingSDGOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingSDGOpDesc.getOutputSchemas" should diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala index 88a611af728..6b7476d7a3b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala @@ -44,7 +44,7 @@ class SklearnTrainingSVMOpDescSpec extends AnyFlatSpec with Matchers { d.countVectorizer shouldBe false d.tfidfTransformer shouldBe false d.target shouldBe null - d.text shouldBe null + d.text shouldBe empty } "SklearnTrainingSVMOpDesc.getOutputSchemas" should