Update to version 2.1.0.

Submitted by:	Mark Dixon <mnd999@gmail.com>

3 files changed, 356 insertions, 55 deletions

diff --git a/devel/spark/Makefile b/devel/spark/Makefile
index 1a9c16a..f984005 100644
--- a/devel/spark/Makefile
+++ b/devel/spark/Makefile
@@ -2,7 +2,7 @@
 # $FreeBSD$
 
 PORTNAME=	spark
-PORTVERSION=	1.6.1
+PORTVERSION=	2.1.0
 CATEGORIES=	devel java
 MASTER_SITES=	APACHE/${PORTNAME}/${PORTNAME}-${PORTVERSION} \
 		LOCAL/demon/:maven \
@@ -52,18 +52,18 @@ do-build:
 		-Dmaven.repo.local=${WRKDIR}/m2 clean package \
 		-Dhadoop.version=2.7.2 -Pyarn -Phadoop-2.6 -DskipTests \
 		-Duser.home=${WRKDIR}
-	${JAR} uvf ${WRKSRC}/assembly/target/scala*/*assembly*hadoop*.jar \
+	${JAR} uvf ${WRKSRC}/assembly/target/scala*/jars/snappy-java-*.jar \
 		-C ${WRKDIR}/snappy org
 
 post-build:
 	${RM} ${WRKSRC}/bin/*.cmd ${WRKSRC}/sbin/spark-daemon.sh.orig
 
 do-install:
-	${MKDIR} ${STAGEDIR}${DATADIR}/lib ${STAGEDIR}${DATADIR}/examples ${STAGEDIR}${DATADIR}/bin ${STAGEDIR}${DATADIR}/sbin ${STAGEDIR}${DATADIR}/conf
+	${MKDIR} ${STAGEDIR}${DATADIR}/lib ${STAGEDIR}${DATADIR}/examples/jars ${STAGEDIR}${DATADIR}/bin ${STAGEDIR}${DATADIR}/sbin ${STAGEDIR}${DATADIR}/conf
 	${ECHO_CMD} "Spark ${PORTVERSION} built for Hadoop 2.7.2" > ${STAGEDIR}${DATADIR}/RELEASE
-	${INSTALL_DATA} ${WRKSRC}/assembly/target/scala*/*assembly*hadoop*.jar ${STAGEDIR}${DATADIR}/lib/
-	${INSTALL_DATA} ${WRKSRC}/examples/target/scala*/spark-examples*.jar ${STAGEDIR}${DATADIR}/lib/
-	cd ${WRKSRC}/examples && ${COPYTREE_SHARE} src ${STAGEDIR}${DATADIR}/examples/
+	(cd  ${WRKSRC}/assembly/target/scala* && ${COPYTREE_SHARE} jars  ${STAGEDIR}${DATADIR})
+	${INSTALL_DATA} ${WRKSRC}/examples/target/spark-examples*.jar ${STAGEDIR}${DATADIR}/examples/jars
+	cd ${WRKSRC}/examples && ${COPYTREE_SHARE} src ${STAGEDIR}${DATADIR}/examples
 	cd ${WRKSRC}/bin && ${INSTALL_SCRIPT} * ${STAGEDIR}${DATADIR}/bin/
 	cd ${WRKSRC}/sbin && ${INSTALL_SCRIPT} * ${STAGEDIR}${DATADIR}/sbin/
 	cd ${WRKSRC} && ${COPYTREE_SHARE} python ${STAGEDIR}${DATADIR}/
diff --git a/devel/spark/distinfo b/devel/spark/distinfo
index 2b607a1..581ab19 100644
--- a/devel/spark/distinfo
+++ b/devel/spark/distinfo
@@ -1,4 +1,5 @@
-SHA256 (hadoop/spark-1.6.1.tgz) = 5b7b6a0332449ce74b3e595fe2dfe0900c69fc72677db18f01f2a637bc84c81c
-SIZE (hadoop/spark-1.6.1.tgz) = 12261556
-SHA256 (hadoop/FreeBSD-spark-1.6.1-maven-repository.tar.gz) = 1f28471e507f8b9a60a4f61c4b000bdabf54d364e06249e44e5ea6c75e486c1d
-SIZE (hadoop/FreeBSD-spark-1.6.1-maven-repository.tar.gz) = 316286846
+TIMESTAMP = 1490618705
+SHA256 (hadoop/spark-2.1.0.tgz) = 8904a7c8be23006cfe2341409ec54b0c53caa0f8baf5f06efc0721f1e0113b38
+SIZE (hadoop/spark-2.1.0.tgz) = 13120365
+SHA256 (hadoop/FreeBSD-spark-2.1.0-maven-repository.tar.gz) = fbb2c23512abc7f2d501c2e578284382350e9420e022d7c1afcf27fbc9491e58
+SIZE (hadoop/FreeBSD-spark-2.1.0-maven-repository.tar.gz) = 267083508
diff --git a/devel/spark/pkg-plist b/devel/spark/pkg-plist
index 18b430e..462ec60 100644
--- a/devel/spark/pkg-plist
+++ b/devel/spark/pkg-plist
@@ -1,5 +1,6 @@
 %%DATADIR%%/RELEASE
 %%DATADIR%%/bin/beeline
+%%DATADIR%%/bin/find-spark-home
 %%DATADIR%%/bin/load-spark-env.sh
 %%DATADIR%%/bin/pyspark
 %%DATADIR%%/bin/run-example
@@ -15,6 +16,8 @@
 %%DATADIR%%/conf/slaves.template
 %%DATADIR%%/conf/spark-defaults.conf.template
 %%DATADIR%%/conf/spark-env.sh.template
+%%DATADIR%%/examples/jars/spark-examples_2.11-2.1.0-sources.jar
+%%DATADIR%%/examples/jars/spark-examples_2.11-2.1.0-test-sources.jar
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaHdfsLR.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaLogQuery.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaPageRank.java
@@ -23,45 +26,57 @@
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaTC.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaCrossValidatorExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDocument.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaGaussianMixtureExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaGeneralizedLinearRegressionExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaInteractionExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaIsotonicRegressionExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaLabeledDocument.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaMulticlassLogisticRegressionWithElasticNetExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaTrainValidationSplitExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java
@@ -70,19 +85,27 @@
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaBisectingKMeansExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaChiSqSelectorExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeClassificationExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeRegressionExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaFPGrowthExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaElementwiseProductExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingKolmogorovSmirnovTestExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeans.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaKernelDensityEstimationExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLDAExample.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLR.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLinearRegressionWithSGDExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLogisticRegressionWithLBFGSExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaPowerIterationClusteringExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
@@ -90,8 +113,18 @@
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaRegressionMetricsExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVMWithSGDExample.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java
-%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaStreamingTestExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaSummaryStatisticsExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredKafkaWordCount.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCount.java
+%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCountWindowed.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKafkaWordCount.java
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaFlumeEventCount.java
@@ -104,60 +137,87 @@
 %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java
 %%DATADIR%%/examples/src/main/python/als.py
 %%DATADIR%%/examples/src/main/python/avro_inputformat.py
-%%DATADIR%%/examples/src/main/python/cassandra_inputformat.py
-%%DATADIR%%/examples/src/main/python/cassandra_outputformat.py
-%%DATADIR%%/examples/src/main/python/hbase_inputformat.py
-%%DATADIR%%/examples/src/main/python/hbase_outputformat.py
 %%DATADIR%%/examples/src/main/python/kmeans.py
 %%DATADIR%%/examples/src/main/python/logistic_regression.py
 %%DATADIR%%/examples/src/main/python/ml/aft_survival_regression.py
+%%DATADIR%%/examples/src/main/python/ml/als_example.py
 %%DATADIR%%/examples/src/main/python/ml/binarizer_example.py
+%%DATADIR%%/examples/src/main/python/ml/bisecting_k_means_example.py
 %%DATADIR%%/examples/src/main/python/ml/bucketizer_example.py
+%%DATADIR%%/examples/src/main/python/ml/chisq_selector_example.py
+%%DATADIR%%/examples/src/main/python/ml/count_vectorizer_example.py
 %%DATADIR%%/examples/src/main/python/ml/cross_validator.py
 %%DATADIR%%/examples/src/main/python/ml/dataframe_example.py
+%%DATADIR%%/examples/src/main/python/ml/dct_example.py
 %%DATADIR%%/examples/src/main/python/ml/decision_tree_classification_example.py
 %%DATADIR%%/examples/src/main/python/ml/decision_tree_regression_example.py
 %%DATADIR%%/examples/src/main/python/ml/elementwise_product_example.py
+%%DATADIR%%/examples/src/main/python/ml/estimator_transformer_param_example.py
+%%DATADIR%%/examples/src/main/python/ml/gaussian_mixture_example.py
+%%DATADIR%%/examples/src/main/python/ml/generalized_linear_regression_example.py
 %%DATADIR%%/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
 %%DATADIR%%/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
 %%DATADIR%%/examples/src/main/python/ml/index_to_string_example.py
+%%DATADIR%%/examples/src/main/python/ml/isotonic_regression_example.py
 %%DATADIR%%/examples/src/main/python/ml/kmeans_example.py
+%%DATADIR%%/examples/src/main/python/ml/lda_example.py
 %%DATADIR%%/examples/src/main/python/ml/linear_regression_with_elastic_net.py
+%%DATADIR%%/examples/src/main/python/ml/logistic_regression_summary_example.py
 %%DATADIR%%/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
+%%DATADIR%%/examples/src/main/python/ml/max_abs_scaler_example.py
+%%DATADIR%%/examples/src/main/python/ml/min_max_scaler_example.py
+%%DATADIR%%/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
 %%DATADIR%%/examples/src/main/python/ml/multilayer_perceptron_classification.py
 %%DATADIR%%/examples/src/main/python/ml/n_gram_example.py
+%%DATADIR%%/examples/src/main/python/ml/naive_bayes_example.py
 %%DATADIR%%/examples/src/main/python/ml/normalizer_example.py
+%%DATADIR%%/examples/src/main/python/ml/one_vs_rest_example.py
 %%DATADIR%%/examples/src/main/python/ml/onehot_encoder_example.py
 %%DATADIR%%/examples/src/main/python/ml/pca_example.py
+%%DATADIR%%/examples/src/main/python/ml/pipeline_example.py
 %%DATADIR%%/examples/src/main/python/ml/polynomial_expansion_example.py
+%%DATADIR%%/examples/src/main/python/ml/quantile_discretizer_example.py
 %%DATADIR%%/examples/src/main/python/ml/random_forest_classifier_example.py
 %%DATADIR%%/examples/src/main/python/ml/random_forest_regressor_example.py
 %%DATADIR%%/examples/src/main/python/ml/rformula_example.py
-%%DATADIR%%/examples/src/main/python/ml/simple_params_example.py
-%%DATADIR%%/examples/src/main/python/ml/simple_text_classification_pipeline.py
 %%DATADIR%%/examples/src/main/python/ml/sql_transformer.py
 %%DATADIR%%/examples/src/main/python/ml/standard_scaler_example.py
 %%DATADIR%%/examples/src/main/python/ml/stopwords_remover_example.py
 %%DATADIR%%/examples/src/main/python/ml/string_indexer_example.py
 %%DATADIR%%/examples/src/main/python/ml/tf_idf_example.py
 %%DATADIR%%/examples/src/main/python/ml/tokenizer_example.py
+%%DATADIR%%/examples/src/main/python/ml/train_validation_split.py
 %%DATADIR%%/examples/src/main/python/ml/vector_assembler_example.py
 %%DATADIR%%/examples/src/main/python/ml/vector_indexer_example.py
+%%DATADIR%%/examples/src/main/python/ml/vector_slicer_example.py
 %%DATADIR%%/examples/src/main/python/ml/word2vec_example.py
 %%DATADIR%%/examples/src/main/python/mllib/binary_classification_metrics_example.py
+%%DATADIR%%/examples/src/main/python/mllib/bisecting_k_means_example.py
 %%DATADIR%%/examples/src/main/python/mllib/correlations.py
+%%DATADIR%%/examples/src/main/python/mllib/correlations_example.py
 %%DATADIR%%/examples/src/main/python/mllib/decision_tree_classification_example.py
 %%DATADIR%%/examples/src/main/python/mllib/decision_tree_regression_example.py
+%%DATADIR%%/examples/src/main/python/mllib/elementwise_product_example.py
 %%DATADIR%%/examples/src/main/python/mllib/fpgrowth_example.py
+%%DATADIR%%/examples/src/main/python/mllib/gaussian_mixture_example.py
 %%DATADIR%%/examples/src/main/python/mllib/gaussian_mixture_model.py
 %%DATADIR%%/examples/src/main/python/mllib/gradient_boosting_classification_example.py
 %%DATADIR%%/examples/src/main/python/mllib/gradient_boosting_regression_example.py
+%%DATADIR%%/examples/src/main/python/mllib/hypothesis_testing_example.py
+%%DATADIR%%/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
 %%DATADIR%%/examples/src/main/python/mllib/isotonic_regression_example.py
+%%DATADIR%%/examples/src/main/python/mllib/k_means_example.py
+%%DATADIR%%/examples/src/main/python/mllib/kernel_density_estimation_example.py
 %%DATADIR%%/examples/src/main/python/mllib/kmeans.py
+%%DATADIR%%/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
+%%DATADIR%%/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
 %%DATADIR%%/examples/src/main/python/mllib/logistic_regression.py
+%%DATADIR%%/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
 %%DATADIR%%/examples/src/main/python/mllib/multi_class_metrics_example.py
 %%DATADIR%%/examples/src/main/python/mllib/multi_label_metrics_example.py
 %%DATADIR%%/examples/src/main/python/mllib/naive_bayes_example.py
+%%DATADIR%%/examples/src/main/python/mllib/normalizer_example.py
+%%DATADIR%%/examples/src/main/python/mllib/power_iteration_clustering_example.py
 %%DATADIR%%/examples/src/main/python/mllib/random_forest_classification_example.py
 %%DATADIR%%/examples/src/main/python/mllib/random_forest_regression_example.py
 %%DATADIR%%/examples/src/main/python/mllib/random_rdd_generation.py
@@ -165,28 +225,55 @@
 %%DATADIR%%/examples/src/main/python/mllib/recommendation_example.py
 %%DATADIR%%/examples/src/main/python/mllib/regression_metrics_example.py
 %%DATADIR%%/examples/src/main/python/mllib/sampled_rdds.py
+%%DATADIR%%/examples/src/main/python/mllib/standard_scaler_example.py
+%%DATADIR%%/examples/src/main/python/mllib/stratified_sampling_example.py
+%%DATADIR%%/examples/src/main/python/mllib/streaming_k_means_example.py
+%%DATADIR%%/examples/src/main/python/mllib/streaming_linear_regression_example.py
+%%DATADIR%%/examples/src/main/python/mllib/summary_statistics_example.py
+%%DATADIR%%/examples/src/main/python/mllib/svm_with_sgd_example.py
+%%DATADIR%%/examples/src/main/python/mllib/tf_idf_example.py
 %%DATADIR%%/examples/src/main/python/mllib/word2vec.py
+%%DATADIR%%/examples/src/main/python/mllib/word2vec_example.py
 %%DATADIR%%/examples/src/main/python/pagerank.py
 %%DATADIR%%/examples/src/main/python/parquet_inputformat.py
 %%DATADIR%%/examples/src/main/python/pi.py
 %%DATADIR%%/examples/src/main/python/sort.py
-%%DATADIR%%/examples/src/main/python/sql.py
+%%DATADIR%%/examples/src/main/python/sql/basic.py
+%%DATADIR%%/examples/src/main/python/sql/datasource.py
+%%DATADIR%%/examples/src/main/python/sql/hive.py
+%%DATADIR%%/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
+%%DATADIR%%/examples/src/main/python/sql/streaming/structured_network_wordcount.py
+%%DATADIR%%/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
 %%DATADIR%%/examples/src/main/python/status_api_demo.py
 %%DATADIR%%/examples/src/main/python/streaming/direct_kafka_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/flume_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/hdfs_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/kafka_wordcount.py
-%%DATADIR%%/examples/src/main/python/streaming/mqtt_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/network_wordcount.py
+%%DATADIR%%/examples/src/main/python/streaming/network_wordjoinsentiments.py
 %%DATADIR%%/examples/src/main/python/streaming/queue_stream.py
 %%DATADIR%%/examples/src/main/python/streaming/recoverable_network_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/sql_network_wordcount.py
 %%DATADIR%%/examples/src/main/python/streaming/stateful_network_wordcount.py
 %%DATADIR%%/examples/src/main/python/transitive_closure.py
 %%DATADIR%%/examples/src/main/python/wordcount.py
+%%DATADIR%%/examples/src/main/r/RSparkSQLExample.R
 %%DATADIR%%/examples/src/main/r/data-manipulation.R
 %%DATADIR%%/examples/src/main/r/dataframe.R
-%%DATADIR%%/examples/src/main/r/ml.R
+%%DATADIR%%/examples/src/main/r/ml/als.R
+%%DATADIR%%/examples/src/main/r/ml/gaussianMixture.R
+%%DATADIR%%/examples/src/main/r/ml/gbt.R
+%%DATADIR%%/examples/src/main/r/ml/glm.R
+%%DATADIR%%/examples/src/main/r/ml/isoreg.R
+%%DATADIR%%/examples/src/main/r/ml/kmeans.R
+%%DATADIR%%/examples/src/main/r/ml/kstest.R
+%%DATADIR%%/examples/src/main/r/ml/lda.R
+%%DATADIR%%/examples/src/main/r/ml/logit.R
+%%DATADIR%%/examples/src/main/r/ml/ml.R
+%%DATADIR%%/examples/src/main/r/ml/mlp.R
+%%DATADIR%%/examples/src/main/r/ml/naiveBayes.R
+%%DATADIR%%/examples/src/main/r/ml/randomForest.R
+%%DATADIR%%/examples/src/main/r/ml/survreg.R
 %%DATADIR%%/examples/src/main/resources/full_user.avsc
 %%DATADIR%%/examples/src/main/resources/kv1.txt
 %%DATADIR%%/examples/src/main/resources/people.json
@@ -195,13 +282,10 @@
 %%DATADIR%%/examples/src/main/resources/users.avro
 %%DATADIR%%/examples/src/main/resources/users.parquet
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ExceptionHandlingTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/GroupByTest.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/LocalFileLR.scala
@@ -219,17 +303,23 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkPi.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTC.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTachyonPi.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/ConnectedComponentsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/LiveJournalPageRank.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/PageRankExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/SSSPExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/SynthBenchmark.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/TriangleCountingExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/BisectingKMeansExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/CrossValidatorExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala
@@ -237,10 +327,15 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GBTExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GaussianMixtureExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GeneralizedLinearRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/InteractionExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/IsotonicRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionExample.scala
@@ -248,14 +343,20 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionWithElasticNetExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MinHashLSHExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MovieLensALS.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MulticlassLogisticRegressionWithElasticNetExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala
@@ -263,14 +364,12 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/TrainValidationSplitExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala
@@ -280,26 +379,41 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/BisectingKMeansExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/CorrelationsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/CosineSimilarity.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DenseGaussianMixture.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DenseKMeans.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/FPGrowthExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/GaussianMixtureExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostedTreesRunner.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingKolmogorovSmirnovTestExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/KMeansExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/KernelDensityEstimationExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LatentDirichletAllocationExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegressionWithSGDExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LogisticRegressionWithLBFGSExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PCAExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PMMLModelExportExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala
@@ -308,28 +422,36 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetricsExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SVMWithSGDExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SparseNaiveBayes.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StratifiedSamplingExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLinearRegression.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLinearRegressionExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLogisticRegression.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingTestExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SummaryStatisticsExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnyPCA.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnySVD.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/Word2VecExample.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/pythonconverters/CassandraConverters.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/pythonconverters/HBaseConverters.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/ActorWordCount.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredKafkaWordCount.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCount.scala
+%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCountWindowed.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/HdfsWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/KafkaWordCount.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/MQTTWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/QueueStream.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/RawNetworkGrep.scala
@@ -337,15 +459,179 @@
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/TwitterAlgebirdCMS.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/TwitterAlgebirdHLL.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/TwitterPopularTags.scala
-%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/ZeroMQWordCount.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewGenerator.scala
 %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewStream.scala
-%%DATADIR%%/lib/spark-assembly-%%VER%%-hadoop2.7.2.jar
-%%DATADIR%%/lib/spark-examples-%%VER%%-hadoop2.7.2.jar
+%%DATADIR%%/jars/RoaringBitmap-0.5.11.jar
+%%DATADIR%%/jars/activation-1.1.1.jar
+%%DATADIR%%/jars/antlr4-runtime-4.5.3.jar
+%%DATADIR%%/jars/aopalliance-1.0.jar
+%%DATADIR%%/jars/aopalliance-repackaged-2.4.0-b34.jar
+%%DATADIR%%/jars/apacheds-i18n-2.0.0-M15.jar
+%%DATADIR%%/jars/apacheds-kerberos-codec-2.0.0-M15.jar
+%%DATADIR%%/jars/api-asn1-api-1.0.0-M20.jar
+%%DATADIR%%/jars/api-util-1.0.0-M20.jar
+%%DATADIR%%/jars/arpack_combined_all-0.1.jar
+%%DATADIR%%/jars/avro-1.7.7.jar
+%%DATADIR%%/jars/avro-ipc-1.7.7.jar
+%%DATADIR%%/jars/avro-mapred-1.7.7-hadoop2.jar
+%%DATADIR%%/jars/base64-2.3.8.jar
+%%DATADIR%%/jars/bcprov-jdk15on-1.51.jar
+%%DATADIR%%/jars/breeze-macros_2.11-0.12.jar
+%%DATADIR%%/jars/breeze_2.11-0.12.jar
+%%DATADIR%%/jars/chill-java-0.8.0.jar
+%%DATADIR%%/jars/chill_2.11-0.8.0.jar
+%%DATADIR%%/jars/commons-beanutils-1.7.0.jar
+%%DATADIR%%/jars/commons-beanutils-core-1.8.0.jar
+%%DATADIR%%/jars/commons-cli-1.2.jar
+%%DATADIR%%/jars/commons-codec-1.10.jar
+%%DATADIR%%/jars/commons-collections-3.2.2.jar
+%%DATADIR%%/jars/commons-compiler-3.0.0.jar
+%%DATADIR%%/jars/commons-compress-1.4.1.jar
+%%DATADIR%%/jars/commons-configuration-1.6.jar
+%%DATADIR%%/jars/commons-crypto-1.0.0.jar
+%%DATADIR%%/jars/commons-digester-1.8.jar
+%%DATADIR%%/jars/commons-httpclient-3.1.jar
+%%DATADIR%%/jars/commons-io-2.4.jar
+%%DATADIR%%/jars/commons-lang-2.6.jar
+%%DATADIR%%/jars/commons-lang3-3.5.jar
+%%DATADIR%%/jars/commons-math3-3.4.1.jar
+%%DATADIR%%/jars/commons-net-2.2.jar
+%%DATADIR%%/jars/compress-lzf-1.0.3.jar
+%%DATADIR%%/jars/core-1.1.2.jar
+%%DATADIR%%/jars/curator-client-2.6.0.jar
+%%DATADIR%%/jars/curator-framework-2.6.0.jar
+%%DATADIR%%/jars/curator-recipes-2.6.0.jar
+%%DATADIR%%/jars/gson-2.2.4.jar
+%%DATADIR%%/jars/guava-14.0.1.jar
+%%DATADIR%%/jars/guice-3.0.jar
+%%DATADIR%%/jars/guice-servlet-3.0.jar
+%%DATADIR%%/jars/hadoop-annotations-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-auth-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-client-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-common-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-hdfs-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-mapreduce-client-app-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-mapreduce-client-common-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-mapreduce-client-core-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-mapreduce-client-jobclient-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-mapreduce-client-shuffle-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-yarn-api-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-yarn-client-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-yarn-common-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-yarn-server-common-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hadoop-yarn-server-web-proxy-%%PYTHON_VER%%.2.jar
+%%DATADIR%%/jars/hk2-api-2.4.0-b34.jar
+%%DATADIR%%/jars/hk2-locator-2.4.0-b34.jar
+%%DATADIR%%/jars/hk2-utils-2.4.0-b34.jar
+%%DATADIR%%/jars/htrace-core-3.1.0-incubating.jar
+%%DATADIR%%/jars/httpclient-4.5.2.jar
+%%DATADIR%%/jars/httpcore-4.4.4.jar
+%%DATADIR%%/jars/ivy-2.4.0.jar
+%%DATADIR%%/jars/jackson-annotations-2.6.5.jar
+%%DATADIR%%/jars/jackson-core-2.6.5.jar
+%%DATADIR%%/jars/jackson-core-asl-1.9.13.jar
+%%DATADIR%%/jars/jackson-databind-2.6.5.jar
+%%DATADIR%%/jars/jackson-jaxrs-1.9.13.jar
+%%DATADIR%%/jars/jackson-mapper-asl-1.9.13.jar
+%%DATADIR%%/jars/jackson-module-paranamer-2.6.5.jar
+%%DATADIR%%/jars/jackson-module-scala_2.11-2.6.5.jar
+%%DATADIR%%/jars/jackson-xc-1.9.13.jar
+%%DATADIR%%/jars/janino-3.0.0.jar
+%%DATADIR%%/jars/java-xmlbuilder-1.0.jar
+%%DATADIR%%/jars/javassist-3.18.1-GA.jar
+%%DATADIR%%/jars/javax.annotation-api-1.2.jar
+%%DATADIR%%/jars/javax.inject-1.jar
+%%DATADIR%%/jars/javax.inject-2.4.0-b34.jar
+%%DATADIR%%/jars/javax.servlet-api-3.1.0.jar
+%%DATADIR%%/jars/javax.ws.rs-api-2.0.1.jar
+%%DATADIR%%/jars/jaxb-api-2.2.2.jar
+%%DATADIR%%/jars/jcl-over-slf4j-1.7.16.jar
+%%DATADIR%%/jars/jersey-client-2.22.2.jar
+%%DATADIR%%/jars/jersey-common-2.22.2.jar
+%%DATADIR%%/jars/jersey-container-servlet-2.22.2.jar
+%%DATADIR%%/jars/jersey-container-servlet-core-2.22.2.jar
+%%DATADIR%%/jars/jersey-guava-2.22.2.jar
+%%DATADIR%%/jars/jersey-media-jaxb-2.22.2.jar
+%%DATADIR%%/jars/jersey-server-2.22.2.jar
+%%DATADIR%%/jars/jets3t-0.9.3.jar
+%%DATADIR%%/jars/jetty-6.1.26.jar
+%%DATADIR%%/jars/jetty-util-6.1.26.jar
+%%DATADIR%%/jars/jline-2.12.1.jar
+%%DATADIR%%/jars/json4s-ast_2.11-3.2.11.jar
+%%DATADIR%%/jars/json4s-core_2.11-3.2.11.jar
+%%DATADIR%%/jars/json4s-jackson_2.11-3.2.11.jar
+%%DATADIR%%/jars/jsp-api-2.1.jar
+%%DATADIR%%/jars/jsr305-1.3.9.jar
+%%DATADIR%%/jars/jtransforms-2.4.0.jar
+%%DATADIR%%/jars/jul-to-slf4j-1.7.16.jar
+%%DATADIR%%/jars/kryo-shaded-3.0.3.jar
+%%DATADIR%%/jars/leveldbjni-all-1.8.jar
+%%DATADIR%%/jars/log4j-1.2.17.jar
+%%DATADIR%%/jars/lz4-1.3.0.jar
+%%DATADIR%%/jars/mail-1.4.7.jar
+%%DATADIR%%/jars/metrics-core-3.1.2.jar
+%%DATADIR%%/jars/metrics-graphite-3.1.2.jar
+%%DATADIR%%/jars/metrics-json-3.1.2.jar
+%%DATADIR%%/jars/metrics-jvm-3.1.2.jar
+%%DATADIR%%/jars/minlog-1.3.0.jar
+%%DATADIR%%/jars/mx4j-3.0.2.jar
+%%DATADIR%%/jars/netty-3.8.0.Final.jar
+%%DATADIR%%/jars/netty-all-4.0.42.Final.jar
+%%DATADIR%%/jars/objenesis-2.1.jar
+%%DATADIR%%/jars/opencsv-2.3.jar
+%%DATADIR%%/jars/oro-2.0.8.jar
+%%DATADIR%%/jars/osgi-resource-locator-1.0.1.jar
+%%DATADIR%%/jars/paranamer-2.6.jar
+%%DATADIR%%/jars/parquet-column-1.8.1.jar
+%%DATADIR%%/jars/parquet-common-1.8.1.jar
+%%DATADIR%%/jars/parquet-encoding-1.8.1.jar
+%%DATADIR%%/jars/parquet-format-2.3.0-incubating.jar
+%%DATADIR%%/jars/parquet-hadoop-1.8.1.jar
+%%DATADIR%%/jars/parquet-jackson-1.8.1.jar
+%%DATADIR%%/jars/pmml-model-1.2.15.jar
+%%DATADIR%%/jars/pmml-schema-1.2.15.jar
+%%DATADIR%%/jars/protobuf-java-2.5.0.jar
+%%DATADIR%%/jars/py4j-0.10.4.jar
+%%DATADIR%%/jars/pyrolite-4.13.jar
+%%DATADIR%%/jars/scala-compiler-2.11.8.jar
+%%DATADIR%%/jars/scala-library-2.11.8.jar
+%%DATADIR%%/jars/scala-parser-combinators_2.11-1.0.4.jar
+%%DATADIR%%/jars/scala-reflect-2.11.8.jar
+%%DATADIR%%/jars/scala-xml_2.11-1.0.2.jar
+%%DATADIR%%/jars/scalap-2.11.8.jar
+%%DATADIR%%/jars/shapeless_2.11-2.0.0.jar
+%%DATADIR%%/jars/slf4j-api-1.7.16.jar
+%%DATADIR%%/jars/slf4j-log4j12-1.7.16.jar
+%%DATADIR%%/jars/snappy-java-1.1.2.6.jar
+%%DATADIR%%/jars/spark-catalyst_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-core_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-graphx_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-launcher_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-mllib-local_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-mllib_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-network-common_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-network-shuffle_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-repl_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-sketch_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-sql_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-streaming_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-tags_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-unsafe_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spark-yarn_2.11-%%VER%%.jar
+%%DATADIR%%/jars/spire-macros_2.11-0.7.4.jar
+%%DATADIR%%/jars/spire_2.11-0.7.4.jar
+%%DATADIR%%/jars/stax-api-1.0-2.jar
+%%DATADIR%%/jars/stream-%%PYTHON_VER%%.0.jar
+%%DATADIR%%/jars/univocity-parsers-2.2.1.jar
+%%DATADIR%%/jars/validation-api-1.1.0.Final.jar
+%%DATADIR%%/jars/xbean-asm5-shaded-4.4.jar
+%%DATADIR%%/jars/xercesImpl-2.9.1.jar
+%%DATADIR%%/jars/xmlenc-0.52.jar
+%%DATADIR%%/jars/xz-1.0.jar
+%%DATADIR%%/jars/zookeeper-3.4.6.jar
 %%DATADIR%%/python/.gitignore
+%%DATADIR%%/python/MANIFEST.in
+%%DATADIR%%/python/README.md
 %%DATADIR%%/python/docs/Makefile
 %%DATADIR%%/python/docs/_static/pyspark.css
 %%DATADIR%%/python/docs/_static/pyspark.js
@@ -361,8 +647,9 @@
 %%DATADIR%%/python/docs/pyspark.sql.rst
 %%DATADIR%%/python/docs/pyspark.streaming.rst
 %%DATADIR%%/python/lib/PY4J_LICENSE.txt
-%%DATADIR%%/python/lib/py4j-0.9-src.zip
+%%DATADIR%%/python/lib/py4j-0.10.4-src.zip
 %%DATADIR%%/python/lib/pyspark.zip
+%%DATADIR%%/python/pylintrc
 %%DATADIR%%/python/pyspark/__init__.py
 %%DATADIR%%/python/pyspark/accumulators.py
 %%DATADIR%%/python/pyspark/broadcast.py
@@ -371,14 +658,18 @@
 %%DATADIR%%/python/pyspark/context.py
 %%DATADIR%%/python/pyspark/daemon.py
 %%DATADIR%%/python/pyspark/files.py
+%%DATADIR%%/python/pyspark/find_spark_home.py
 %%DATADIR%%/python/pyspark/heapq3.py
 %%DATADIR%%/python/pyspark/java_gateway.py
 %%DATADIR%%/python/pyspark/join.py
 %%DATADIR%%/python/pyspark/ml/__init__.py
+%%DATADIR%%/python/pyspark/ml/base.py
 %%DATADIR%%/python/pyspark/ml/classification.py
 %%DATADIR%%/python/pyspark/ml/clustering.py
+%%DATADIR%%/python/pyspark/ml/common.py
 %%DATADIR%%/python/pyspark/ml/evaluation.py
 %%DATADIR%%/python/pyspark/ml/feature.py
+%%DATADIR%%/python/pyspark/ml/linalg/__init__.py
 %%DATADIR%%/python/pyspark/ml/param/__init__.py
 %%DATADIR%%/python/pyspark/ml/param/_shared_params_code_gen.py
 %%DATADIR%%/python/pyspark/ml/param/shared.py
@@ -417,12 +708,16 @@
 %%DATADIR%%/python/pyspark/shell.py
 %%DATADIR%%/python/pyspark/shuffle.py
 %%DATADIR%%/python/pyspark/sql/__init__.py
+%%DATADIR%%/python/pyspark/sql/catalog.py
 %%DATADIR%%/python/pyspark/sql/column.py
+%%DATADIR%%/python/pyspark/sql/conf.py
 %%DATADIR%%/python/pyspark/sql/context.py
 %%DATADIR%%/python/pyspark/sql/dataframe.py
 %%DATADIR%%/python/pyspark/sql/functions.py
 %%DATADIR%%/python/pyspark/sql/group.py
 %%DATADIR%%/python/pyspark/sql/readwriter.py
+%%DATADIR%%/python/pyspark/sql/session.py
+%%DATADIR%%/python/pyspark/sql/streaming.py
 %%DATADIR%%/python/pyspark/sql/tests.py
 %%DATADIR%%/python/pyspark/sql/types.py
 %%DATADIR%%/python/pyspark/sql/utils.py
@@ -437,16 +732,20 @@
 %%DATADIR%%/python/pyspark/streaming/kafka.py
 %%DATADIR%%/python/pyspark/streaming/kinesis.py
 %%DATADIR%%/python/pyspark/streaming/listener.py
-%%DATADIR%%/python/pyspark/streaming/mqtt.py
 %%DATADIR%%/python/pyspark/streaming/tests.py
 %%DATADIR%%/python/pyspark/streaming/util.py
 %%DATADIR%%/python/pyspark/tests.py
 %%DATADIR%%/python/pyspark/traceback_utils.py
+%%DATADIR%%/python/pyspark/version.py
 %%DATADIR%%/python/pyspark/worker.py
 %%DATADIR%%/python/run-tests
 %%DATADIR%%/python/run-tests.py
+%%DATADIR%%/python/setup.cfg
+%%DATADIR%%/python/setup.py
 %%DATADIR%%/python/test_support/SimpleHTTPServer.py
-%%DATADIR%%/python/test_support/hello.txt
+%%DATADIR%%/python/test_support/hello/hello.txt
+%%DATADIR%%/python/test_support/hello/sub_hello/sub_hello.txt
+%%DATADIR%%/python/test_support/sql/ages.csv
 %%DATADIR%%/python/test_support/sql/orc_partitioned/_SUCCESS
 %%DATADIR%%/python/test_support/sql/orc_partitioned/b=0/c=0/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc
 %%DATADIR%%/python/test_support/sql/orc_partitioned/b=0/c=0/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc
@@ -467,6 +766,7 @@
 %%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=9/day=1/part-r-00007.gz.parquet
 %%DATADIR%%/python/test_support/sql/people.json
 %%DATADIR%%/python/test_support/sql/people1.json
+%%DATADIR%%/python/test_support/sql/streaming/text-test.txt
 %%DATADIR%%/python/test_support/sql/text-test.txt
 %%DATADIR%%/python/test_support/userlib-0.1.zip
 %%DATADIR%%/python/test_support/userlibrary.py