summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSanto Cariotti <santo@dcariotti.me>2025-01-08 13:55:47 +0100
committerSanto Cariotti <santo@dcariotti.me>2025-01-08 13:55:47 +0100
commit975715da5b2c1d31be466b17bc7a25c1999ed28d (patch)
treedf28c86180c753d28ce563daeed272c0bb833540
parent83c3fa60c752486d2b4d0001b6fad3ee31fb9672 (diff)
Use params instead of `read` function
-rwxr-xr-xscripts/01-create-bucket.sh13
-rwxr-xr-xscripts/03-update-network-for-dataproc.sh2
-rwxr-xr-xscripts/04-dataproc-create-cluster.sh24
-rwxr-xr-xscripts/06-dataproc-update-cluster.sh20
4 files changed, 35 insertions, 24 deletions
diff --git a/scripts/01-create-bucket.sh b/scripts/01-create-bucket.sh
index 364e098..30bedc0 100755
--- a/scripts/01-create-bucket.sh
+++ b/scripts/01-create-bucket.sh
@@ -1,17 +1,18 @@
#!/bin/sh
-path=$(find . -name 'order_products.csv' -print -quit)
+path="$1"
-if [ -z "$path" ]; then
- read -p "Enter 'order_products.csv' path: " path
- if [ -z "$path" ] || [ ! -e "$path" ]; then
+if [ -z "$path" ] || [ ! -e "$path" ]; then
+ path=$(find . -name 'order_products.csv' -print -quit)
+
+ if [ -z "$path" ]; then
echo ">>>> Path not found..."
exit 1
fi
fi
-gcloud storage buckets create gs://$BUCKET_NAME --location=eu
+gcloud storage buckets create gs://${BUCKET_NAME} --location=eu
gcloud storage buckets add-iam-policy-binding gs://${BUCKET_NAME} \
--member="serviceAccount:${SERVICE_ACCOUNT}@${PROJECT}.iam.gserviceaccount.com" \
--role="roles/storage.objectCreator"
-gcloud storage cp $path gs://$BUCKET_NAME/input/
+gcloud storage cp $path gs://${BUCKET_NAME}/input/
diff --git a/scripts/03-update-network-for-dataproc.sh b/scripts/03-update-network-for-dataproc.sh
index 2d27945..c091f22 100755
--- a/scripts/03-update-network-for-dataproc.sh
+++ b/scripts/03-update-network-for-dataproc.sh
@@ -1,6 +1,6 @@
#!/bin/sh
gcloud compute networks subnets update default \
- --region $REGION \
+ --region=${REGION} \
--enable-private-ip-google-access
diff --git a/scripts/04-dataproc-create-cluster.sh b/scripts/04-dataproc-create-cluster.sh
index d2dd080..8bdeccb 100755
--- a/scripts/04-dataproc-create-cluster.sh
+++ b/scripts/04-dataproc-create-cluster.sh
@@ -2,29 +2,28 @@
set -eu
-if [ "$#" -ne 1 ]; then
- echo "Usage: 'sh ${PWD}/$0 <num-workers>'"
+if [ "$#" -ne 3 ]; then
+ echo "Usage: 'sh ${PWD}/$0 <num-workers> <master-machine> <worker-machine>'"
exit 1
fi
NUM_WORKERS="$1"
+MASTER_MACHINE="$2"
+WORKER_MACHINE="$3"
+
if [ "$NUM_WORKERS" -lt 1 ] || [ "$NUM_WORKERS" -gt 4 ]; then
echo "<num-workers> must be 1, 2, 3, or 4"
exit 1
fi
-read -p "Enter worker-machine-type: " worker_machine
-
-if [ -z "$worker_machine" ]; then
- echo "Error: Worker machine type cannot be empty."
+if [ -z "$MASTER_MACHINE" ]; then
+ echo "Error: Master machine type cannot be empty."
exit 1
fi
-read -p "Enter master-machine-type: " master_machine
-
-if [ -z "$master_machine" ]; then
- echo "Error: Master machine type cannot be empty."
+if [ -z "$WORKER_MACHINE" ]; then
+ echo "Error: Worker machine type cannot be empty."
exit 1
fi
@@ -34,9 +33,8 @@ COMMON_PARAMS="\
--service-account=${SERVICE_ACCOUNT}@${PROJECT}.iam.gserviceaccount.com \
--master-boot-disk-size=240 \
--worker-boot-disk-size=240 \
- --worker-machine-type=${worker_machine} \
- --master-machine-type=${master_machine}"
-
+ --worker-machine-type=${WORKER_MACHINE} \
+ --master-machine-type=${MASTER_MACHINE}"
if [ "$NUM_WORKERS" -eq 1 ]; then
echo ">>>> Creating a single-node cluster..."
diff --git a/scripts/06-dataproc-update-cluster.sh b/scripts/06-dataproc-update-cluster.sh
index cb098ef..8714e7d 100755
--- a/scripts/06-dataproc-update-cluster.sh
+++ b/scripts/06-dataproc-update-cluster.sh
@@ -2,18 +2,30 @@
set -eu
-if [ "$#" -ne 1 ]; then
- echo "Usage: 'sh ${PWD}/$0 <num-workers>'"
+if [ "$#" -ne 3 ]; then
+ echo "Usage: 'sh ${PWD}/$0 <num-workers> <master-machine> <worker-machine>'"
exit 1
fi
NUM_WORKERS="$1"
+MASTER_MACHINE="$2"
+WORKER_MACHINE="$3"
if [ "$NUM_WORKERS" -lt 1 ] || [ "$NUM_WORKERS" -gt 4 ]; then
echo "<num-workers> must be 1, 2, 3, or 4"
exit 1
fi
+if [ -z "$MASTER_MACHINE" ]; then
+ echo "Error: Master machine type cannot be empty."
+ exit 1
+fi
+
+if [ -z "$WORKER_MACHINE" ]; then
+ echo "Error: Worker machine type cannot be empty."
+ exit 1
+fi
+
# Handle single worker case
if [ "$NUM_WORKERS" -eq 1 ]; then
@@ -23,7 +35,7 @@ if [ "$NUM_WORKERS" -eq 1 ]; then
fi
echo ">>>> Creating a new cluster with 1 worker..."
- eval "scripts/04-dataproc-create-cluster.sh 1"
+ eval "scripts/04-dataproc-create-cluster.sh 1 ${MASTER_MACHINE} ${WORKER_MACHINE}"
else
if ! gcloud dataproc clusters update "${CLUSTER}" \
--project="${PROJECT}" --region="${REGION}" \
@@ -32,7 +44,7 @@ else
gcloud dataproc clusters delete "${CLUSTER}" --region="${REGION}" --quiet
echo ">>>> Creating a new cluster with ${NUM_WORKERS} workers..."
- eval "scripts/04-dataproc-create-cluster.sh ${NUM_WORKERS}"
+ eval "scripts/04-dataproc-create-cluster.sh ${NUM_WORKERS} ${MASTER_MACHINE} ${WORKER_MACHINE}"
else
echo ">>>> Successfully updated the cluster to ${NUM_WORKERS} workers."
fi