Spark: remove the service dependance to HDFS in configuring process (#…

…1623)
oap-project · Jul 5, 2023 · 77046db · 77046db
1 parent f5c5dda
commit 77046db
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 13 deletions.
diff --git a/python/cloudtik/runtime/hdfs/scripts/configure.sh b/python/cloudtik/runtime/hdfs/scripts/configure.sh
@@ -94,7 +94,7 @@ function configure_hdfs() {
 
     if [ $IS_HEAD_NODE == "true" ];then
         # TODO: format only once if there is no force format flag
-        export HADOOP_CONF_DIR= ${HDFS_CONF_DIR}
+        export HADOOP_CONF_DIR=${HDFS_CONF_DIR}
         # Stop namenode in case it was running left from last try
         ${HADOOP_HOME}/bin/hdfs --daemon stop namenode > /dev/null 2>&1
         # Format hdfs once

diff --git a/python/cloudtik/runtime/hdfs/scripts/services.sh b/python/cloudtik/runtime/hdfs/scripts/services.sh
@@ -19,7 +19,7 @@ set_head_option "$@"
 set_service_command "$@"
 
 # HDFS use its own conf dir
-export HADOOP_CONF_DIR= ${HADOOP_HOME}/etc/hdfs
+export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hdfs
 
 case "$SERVICE_COMMAND" in
 start)

diff --git a/python/cloudtik/runtime/spark/scripts/configure.sh b/python/cloudtik/runtime/spark/scripts/configure.sh
@@ -377,17 +377,6 @@ function configure_hadoop_and_spark() {
         update_metastore_config
 
         cp -r ${output_dir}/spark/* ${SPARK_HOME}/conf
-
-        if [ "$HDFS_ENABLED" == "true" ]; then
-            # Create event log dir on hdfs
-            ${HADOOP_HOME}/bin/hdfs --loglevel WARN --daemon start namenode
-            ${HADOOP_HOME}/bin/hadoop --loglevel WARN fs -mkdir -p /shared/spark-events
-            ${HADOOP_HOME}/bin/hdfs --loglevel WARN --daemon stop namenode
-        else
-            # Create event log dir on cloud storage if needed
-            # This needs to be done after hadoop file system has been configured correctly
-            ${HADOOP_HOME}/bin/hadoop --loglevel WARN fs -mkdir -p /shared/spark-events
-        fi
     fi
 }
 

diff --git a/python/cloudtik/runtime/spark/scripts/services.sh b/python/cloudtik/runtime/spark/scripts/services.sh
@@ -38,6 +38,10 @@ start)
     mount_cloud_fs
 
     if [ $IS_HEAD_NODE == "true" ]; then
+        # Create event log dir on cloud storage if needed
+        # This needs to be done after hadoop file system has been configured correctly
+        ${HADOOP_HOME}/bin/hadoop --loglevel WARN fs -mkdir -p /shared/spark-events
+
         echo "Starting Resource Manager..."
         $HADOOP_HOME/bin/yarn --daemon start resourcemanager
         echo "Starting Spark History Server..."