Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
ed932d2
Temporarily renames Dataset to DS
liancheng Mar 1, 2016
e59e940
Renames DataFrame to Dataset[T]
liancheng Mar 1, 2016
b357371
Fixes Java API compilation failures
liancheng Mar 1, 2016
3783e31
Fixes styling issues
liancheng Mar 1, 2016
a02a922
Fixes compilation failure introduced while rebasing
liancheng Mar 1, 2016
3db81f8
Temporarily disables MiMA check for convenience
liancheng Mar 1, 2016
f67f497
Fixes infinite recursion in Dataset constructor
liancheng Mar 1, 2016
f921583
Fixes test failures
liancheng Mar 3, 2016
fa22261
Migrates encoder stuff to the new Dataset
liancheng Mar 3, 2016
8cf5672
Makes some shape-keeping operations typed
liancheng Mar 5, 2016
712ee19
Adds collectRows() for Java API
liancheng Mar 6, 2016
c73b91f
Migrates joinWith operations
liancheng Mar 6, 2016
54cb36a
Migrates typed select
liancheng Mar 7, 2016
cbd7519
Renames typed groupBy to groupByKey
liancheng Mar 7, 2016
f1a2903
Migrates typed groupBy
liancheng Mar 7, 2016
15b4193
Migrates functional transformers
liancheng Mar 7, 2016
9aff0e2
Removes the old DS class and gets everything compiled
liancheng Mar 7, 2016
f053852
Fixes compilation error
liancheng Mar 7, 2016
3a7aff4
Row encoder should produce GenericRowWithSchema
liancheng Mar 8, 2016
9f8e0ad
Fixes compilation error after rebasing
liancheng Mar 8, 2016
bc081a9
Migrated Dataset.showString should handle typed objects
liancheng Mar 8, 2016
6b69f43
MapObjects should also handle DecimalType and DateType
liancheng Mar 8, 2016
29cb70f
Always use eager analysis
liancheng Mar 9, 2016
ba86e09
Fixes compilation error after rebasing
liancheng Mar 10, 2016
5727f48
Fixes compilation error after rebasing
liancheng Mar 10, 2016
cd63810
Temporarily ignores Python UDT test cases
liancheng Mar 10, 2016
4c8d139
fix python
cloud-fan Mar 10, 2016
cf0c112
Merge pull request #9 from cloud-fan/ds-to-df
liancheng Mar 10, 2016
91942cf
fix pymllib
cloud-fan Mar 10, 2016
736fbcb
Merge pull request #10 from cloud-fan/ds-to-df
liancheng Mar 10, 2016
488a82e
MIMA
yhuai Mar 10, 2016
343c611
Fix typo...
yhuai Mar 10, 2016
63d4d69
MIMA: Exclude DataFrame class.
yhuai Mar 10, 2016
f6bcd50
Revert "MIMA: Exclude DataFrame class."
yhuai Mar 10, 2016
49c6fc2
Revert "Fix typo..."
yhuai Mar 10, 2016
d52ce17
Revert "MIMA"
yhuai Mar 10, 2016
7d29c06
Merge remote-tracking branch 'upstream/master' into ds-to-df
yhuai Mar 11, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixes compilation error after rebasing
  • Loading branch information
liancheng committed Mar 10, 2016
commit 5727f480f62abecdab3f8c9a3d49209528e4a394
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.ml.feature;

import java.util.Arrays;
import java.util.List;

import org.junit.After;
import org.junit.Assert;
Expand Down Expand Up @@ -70,7 +71,7 @@ public void bucketizerTest() {
.setOutputCol("result")
.setSplits(splits);

Row[] result = bucketizer.transform(dataset).select("result").collect();
List<Row> result = bucketizer.transform(dataset).select("result").collectAsList();

for (Row r : result) {
double index = r.getDouble(0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.ml.feature;

import java.util.Arrays;
import java.util.List;

import edu.emory.mathcs.jtransforms.dct.DoubleDCT_1D;
import org.junit.After;
Expand Down Expand Up @@ -69,8 +70,8 @@ public void javaCompatibilityTest() {
.setInputCol("vec")
.setOutputCol("resultVec");

Row[] result = dct.transform(dataset).select("resultVec").collect();
Vector resultVec = result[0].getAs("resultVec");
List<Row> result = dct.transform(dataset).select("resultVec").collectAsList();
Vector resultVec = result.get(0).getAs("resultVec");

Assert.assertArrayEquals(expectedResult, resultVec.toArray(), 1e-6);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public void hashingTF() {
IDF idf = new IDF().setInputCol("rawFeatures").setOutputCol("features");
IDFModel idfModel = idf.fit(featurizedData);
Dataset<Row> rescaledData = idfModel.transform(featurizedData);
for (Row r : rescaledData.select("features", "label").take(3)) {
for (Row r : rescaledData.select("features", "label").takeAsList(3)) {
Vector features = r.getAs(0);
Assert.assertEquals(features.size(), numFeatures);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ public void polynomialExpansionTest() {

Dataset<Row> dataset = jsql.createDataFrame(data, schema);

Row[] pairs = polyExpansion.transform(dataset)
List<Row> pairs = polyExpansion.transform(dataset)
.select("polyFeatures", "expected")
.collect();
.collectAsList();

for (Row r : pairs) {
double[] polyFeatures = ((Vector)r.get(0)).toArray();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ public void testStringIndexer() {
.setOutputCol("labelIndex");
Dataset<Row> output = indexer.fit(dataset).transform(dataset);

Assert.assertArrayEquals(
new Row[] { cr(0, 0.0), cr(1, 2.0), cr(2, 1.0), cr(3, 0.0), cr(4, 0.0), cr(5, 1.0) },
output.orderBy("id").select("id", "labelIndex").collect());
Assert.assertEquals(
Arrays.asList(cr(0, 0.0), cr(1, 2.0), cr(2, 1.0), cr(3, 0.0), cr(4, 0.0), cr(5, 1.0)),
output.orderBy("id").select("id", "labelIndex").collectAsList());
}

/** An alias for RowFactory.create. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.ml.feature;

import java.util.Arrays;
import java.util.List;

import org.junit.After;
import org.junit.Assert;
Expand Down Expand Up @@ -63,9 +64,9 @@ public void regexTokenizer() {
));
Dataset<Row> dataset = jsql.createDataFrame(rdd, TokenizerTestData.class);

Row[] pairs = myRegExTokenizer.transform(dataset)
List<Row> pairs = myRegExTokenizer.transform(dataset)
.select("tokens", "wantedTokens")
.collect();
.collectAsList();

for (Row r : pairs) {
Assert.assertEquals(r.get(0), r.get(1));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public void testJavaWord2Vec() {
Word2VecModel model = word2Vec.fit(documentDF);
Dataset<Row> result = model.transform(documentDF);

for (Row r: result.select("result").collect()) {
for (Row r: result.select("result").collectAsList()) {
double[] polyFeatures = ((Vector)r.get(0)).toArray();
Assert.assertEquals(polyFeatures.length, 3);
}
Expand Down