summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSanto Cariotti <santo@dcariotti.me>2024-12-27 22:20:18 +0100
committerSanto Cariotti <santo@dcariotti.me>2024-12-27 22:21:59 +0100
commitfc35b02c84c94d5965a8317df6312c9a3ececc68 (patch)
tree9f994efd7eaaaaeafd2d7e268208032ecad94270
parent57078f758729affaab82c7a2d50c5bf4d942e69d (diff)
Fix creds on Main class
-rw-r--r--co-purchase-analysis/src/main/scala/Main.scala20
1 files changed, 13 insertions, 7 deletions
diff --git a/co-purchase-analysis/src/main/scala/Main.scala b/co-purchase-analysis/src/main/scala/Main.scala
index ec113b3..8c3e907 100644
--- a/co-purchase-analysis/src/main/scala/Main.scala
+++ b/co-purchase-analysis/src/main/scala/Main.scala
@@ -75,15 +75,21 @@ object CoPurchaseAnalysis {
* Configured SparkSession instance
*/
def createSparkSession(appName: String, master: String): SparkSession = {
- SparkSession.builder
+ var session = SparkSession.builder
.appName(appName)
.config("spark.master", master)
- .config("spark.hadoop.google.cloud.auth.service.account.enable", "true")
- .config(
- "spark.hadoop.google.cloud.auth.service.account.json.keyfile",
- System.getenv("GOOGLE_APPLICATION_CREDENTIALS")
- )
- .getOrCreate()
+
+ val creds = System.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+ if (creds != null) {
+ session
+ .config("spark.hadoop.google.cloud.auth.service.account.enable", "true")
+ .config(
+ "spark.hadoop.google.cloud.auth.service.account.json.keyfile",
+ creds
+ )
+ }
+
+ session.getOrCreate()
}
/** Parses a single line from the input file into an OrderProduct instance.