From 2aa010e47387f5c60d63824dce65f76f22eecddc Mon Sep 17 00:00:00 2001
From: Santo Cariotti <santo@dcariotti.me>
Date: Sat, 28 Dec 2024 15:17:04 +0100
Subject: Check on scripts + update num worker

---
 scripts/04-dataproc-create-cluster.sh | 38 +++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100755 scripts/04-dataproc-create-cluster.sh

(limited to 'scripts/04-dataproc-create-cluster.sh')

diff --git a/scripts/04-dataproc-create-cluster.sh b/scripts/04-dataproc-create-cluster.sh
new file mode 100755
index 0000000..ada258d
--- /dev/null
+++ b/scripts/04-dataproc-create-cluster.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+set -eu
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: 'sh ${PWD}/$0 <num-workers>'"
+    exit 1
+fi
+
+
+NUM_WORKERS="$1"
+if [ "$NUM_WORKERS" -lt 1 ] || [ "$NUM_WORKERS" -gt 4 ]; then
+    echo "<num-workers> must be 1, 2, 3, or 4"
+    exit 1
+fi
+
+
+COMMON_PARAMS="\
+    --project=${PROJECT} \
+    --region=${REGION} \
+    --service-account=${SERVICE_ACCOUNT}@${PROJECT}.iam.gserviceaccount.com \
+    --master-boot-disk-size=240 \
+    --worker-boot-disk-size=240 \
+    --worker-machine-type=n1-standard-2 \
+    --master-machine-type=n1-standard-2"
+
+
+if [ "$NUM_WORKERS" -eq 1 ]; then
+    echo ">>>> Creating a single-node cluster..."
+    gcloud dataproc clusters create "${CLUSTER}" \
+        ${COMMON_PARAMS} \
+        --single-node
+else
+    echo ">>>> Creating a cluster with ${NUM_WORKERS} workers..."
+    gcloud dataproc clusters create "${CLUSTER}" \
+        ${COMMON_PARAMS} \
+        --num-workers="${NUM_WORKERS}"
+fi
-- 
cgit v1.2.3-71-g8e6c