implement spark 2.0 #299
Description
This is a list of changes for spark 2.0.
Attached is the complete diff of generated JS between 1.6 and 2.0
New Files:
./Accumulable.scala
./Accumulator.scala
./ml/clustering/BisectingKMeans.js
./ml/clustering/BisectingKMeansModel.js
./ml/clustering/GaussianMixture.js
./ml/clustering/GaussianMixtureModel.js
./ml/clustering/GaussianMixtureSummary.js
./ml/clustering/KMeansSummary.js
./ml/feature/LabeledPoint.js
./ml/feature/MaxAbsScaler.js
./ml/feature/MaxAbsScalerModel.js
./ml/regression/GeneralizedLinearRegression.js
./ml/regression/GeneralizedLinearRegressionModel.js
./ml/regression/GeneralizedLinearRegressionSummary.js
./ml/regression/GeneralizedLinearRegressionTrainingSummary.js
./sql/catalog/Catalog.js
./sql/catalog/Column.js
./sql/catalog/Database.js
./sql/catalog/Function.js
./sql/catalog/Table.js
./sql/SparkSession.js
./sql/streaming/DataStreamReader.js
./sql/streaming/DataStreamWriter.js
./sql/streaming/ProcessingTime.js
./sql/streaming/SinkStatus.js
./sql/streaming/SourceStatus.js
./sql/streaming/StreamingQuery.js
./sql/streaming/StreamingQueryException.js
./sql/streaming/StreamingQueryInfo.js
./sql/streaming/StreamingQueryListener.js
./sql/streaming/StreamingQueryManager.js
./sql/streaming/Trigger.js
Removed Files:
./Accumulators.js
./SparkContext.js
./broadcast/HttpBroadcastFactory.js
./sql/DataFrame.js
./sql/DataFrameHolder.js
Modified:
This list does not contain modifications which were just doc changes.
The pervasive general change is DataFrame is now Dataset
Accumulable.js
SparkContext.js
/ml/Estimator.js
ml/Pipeline.js
ml/PipelineModel.js
ml/Transformer.js
ml/UnaryTransformer.js
ml/classification/DecisionTreeClassificationModel.js
ml/classification/GBTClassificationModel.js
ml/classification/GBTClassifier.js
ml/classification/LogisticRegressionModel.js
ml/classification/MultilayerPerceptronClassificationModel.js
ml/classification/ProbabilisticClassificationModel.js
ml/classification/RandomForestClassificationModel.js
ml/classification/RandomForestClassifier.js
ml/clustering/DistributedLDAModel.js
ml/clustering/KMeans.js
ml/clustering/KMeansModel.js
ml/clustering/LDAModel.js
ml/evaluation/Evaluator.js
ml/evaluation/MulticlassClassificationEvaluator.js
ml/evaluation/RegressionEvaluator.js
ml/feature/Binarizer.js
ml/feature/Bucketizer.js
ml/feature/ChiSqSelector.js
ml/feature/ChiSqSelectorModel.js
ml/feature/CountVectorizer.js
ml/feature/CountVectorizerModel.js
ml/feature/ElementwiseProduct.js
ml/feature/IDF.js
ml/feature/IDFModel.js
ml/feature/IndexToString.js
ml/feature/Interaction.js
ml/feature/MinMaxScaler.js
ml/feature/MinMaxScalerModel.js
ml/feature/OneHotEncoder.js
ml/feature/PCAModel.js
ml/feature/QuantileDiscretizer.js
ml/feature/RFormula.js
ml/feature/RFormulaModel.js
ml/feature/SQLTransformer.js
ml/feature/StandardScaler.js
ml/feature/StandardScalerModel.js
ml/feature/StopWordsRemover.js
ml/feature/StringIndexer.js
ml/feature/StringIndexerModel.js
ml/feature/VectorAssembler.js
ml/feature/VectorIndexer.js
ml/feature/VectorIndexerModel.js
ml/feature/VectorSlicer.js
ml/feature/Word2Vec.js
ml/feature/Word2VecModel.js
ml/recommendation/ALS.js
/ml/recommendation/ALSModel.js
ml/regression/AFTSurvivalRegression.js
ml/regression/AFTSurvivalRegressionModel.js
ml/regression/DecisionTreeRegressionModel.js
ml/regression/DecisionTreeRegressor.js
ml/regression/GBTRegressionModel.js
ml/regression/GBTRegressor.js
ml/regression/IsotonicRegression.js
ml/regression/IsotonicRegressionModel.js
ml/regression/LinearRegression.js
ml/regression/LinearRegressionModel.js
ml/regression/RandomForestRegressionModel.js
ml/regression/RandomForestRegressor.js
ml/tuning/CrossValidator.js
ml/tuning/CrossValidatorModel.js
ml/tuning/TrainValidationSplit.js
ml/tuning/TrainValidationSplitModel.js
ml/util/MLReader.js
ml/util/MLWriter.js
mllib/classification/LogisticRegressionWithLBFGS.js
mllib/clustering/BisectingKMeansModel.js
mllib/clustering/EMLDAOptimizer.js
mllib/clustering/LocalLDAModel.js
mllib/evaluation/RegressionMetrics.js
mllib/feature/HashingTF.js
mllib/feature/StandardScalerModel.js
mllib/feature/VectorTransformer.js
mllib/fpm/FPGrowthModel.js
mllib/fpm/PrefixSpanModel.js
mllib/linalg/DenseMatrix.js
mllib/linalg/DenseVector.js
mllib/linalg/Matrices.js
mllib/linalg/Matrix.js
mllib/linalg/SparseMatrix.js
mllib/linalg/SparseVector.js
mllib/linalg/Vector.js
mllib/linalg/VectorUDT.js
mllib/linalg/Vectors.js
mllib/linalg/distributed/RowMatrix.js
mllib/optimization/GradientDescent.js
mllib/random/RandomRDDs.js
mllib/recommendation/ALS.js
mllib/regression/StreamingLinearRegressionWithSGD.js
mllib/tree/DecisionTree.js
mllib/tree/model/GradientBoostedTreesModel.js
mllib/util/MLUtils.js
rdd/PairRDDFunctions.js
rdd/RDD.js
sql/Column.js
sql/DataFrameReader.js
sql/DataFrameWriter.js
sql/Dataset.js
sql/SQLContext.js
sql/UDFRegistration.js
streaming/StreamingContext.js
streaming/api/java/JavaStreamingContext.js
New Examples:
examples/ml/JavaDocument.java
examples/ml/JavaEstimatorTransformerParamExample.java
examples/ml/JavaGaussianMixtureExample.java
examples/ml/JavaGeneralizedLinearRegressionExample.java
examples/ml/JavaLabeledDocument.java
examples/ml/JavaMaxAbsScalerExample.java
examples/ml/JavaModelSelectionViaCrossValidationExample.java
examples/mllib/JavaChiSqSelectorExample.java
examples/mllib/JavaCorrelationsExample.java
examples/mllib/JavaElementwiseProductExample.java
examples/mllib/JavaGaussianMixtureExample.java
examples/mllib/JavaHypothesisTestingExample.java
examples/mllib/JavaHypothesisTestingKolmogorovSmirnovTestExample.java
examples/mllib/JavaKernelDensityEstimationExample.java
examples/mllib/JavaLatentDirichletAllocationExample.java
examples/mllib/JavaLinearRegressionWithSGDExample.java
examples/mllib/JavaLogisticRegressionWithLBFGSExample.java
examples/mllib/JavaPCAExample.java
examples/mllib/JavaStratifiedSamplingExample.java
examples/mllib/JavaStreamingTestExample.java
examples/mllib/JavaSummaryStatisticsExample.java
examples/sql/JavaSQLDataSourceExample.java
examples/sql/hive/JavaSparkHiveExample.java
examples/sql/streaming/JavaStructuredNetworkWordCount.java
examples/sql/streaming/JavaStructuredNetworkWordCountWindowed.java
Deleted Examples:
examples/ml/JavaSimpleParamsExample.java
examples/ml/JavaSimpleTextClassificationPipeline.java
examples/ml/JavaTrainValidationSplitExample.java
examples/mllib/JavaFPGrowthExample.java
examples/mllib/JavaKMeans.java
examples/mllib/JavaLDAExample.java
examples/mllib/JavaLR.java
examples/stream (some now in sql)
almost all examples changed to use SparkSession()