diff options
| -rw-r--r-- | co-purchase-analysis/src/main/scala/Main.scala | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/co-purchase-analysis/src/main/scala/Main.scala b/co-purchase-analysis/src/main/scala/Main.scala index fbca3ac..6f9157e 100644 --- a/co-purchase-analysis/src/main/scala/Main.scala +++ b/co-purchase-analysis/src/main/scala/Main.scala @@ -153,11 +153,11 @@ object CoPurchaseAnalysis { .map(parseLine) val cores = spark.conf.get("spark.executor.cores", "4").toInt - val nodes = spark.conf.get("spark.executor.instance", "4").toInt - val partitinosNumber = + val nodes = spark.conf.get("spark.executor.instances", "4").toInt + val partitionsNumber = math.max(cores * nodes * 2, spark.sparkContext.defaultParallelism * 2) - val result = processData(inputRDD, partitinosNumber) + val result = processData(inputRDD, partitionsNumber) .saveAsTextFile(config("outputPath")) } finally { spark.stop() |