Runtime: set local DNS server as system default resolver

python/cloudtik/core/_private/util/resolv_conf.py

@@ -0,0 +1,65 @@
+import re
+import shutil
+import tempfile
+
+SYSTEM_RESOLV_CONF = "/etc/resolv.conf"
+
+
+def update_resolv_conf(name_servers, resolv_conf=None):
+    if not resolv_conf:
+        resolv_conf = SYSTEM_RESOLV_CONF
+
+    old_name_servers, search, sort_list, options = parse_resolv_conf(
+        resolv_conf)
+
+    # write to temp file first
+    resolv_conf_temp = tempfile.mktemp(prefix=f"resolv.conf_")
+    with open(resolv_conf_temp, "w") as f:
+        for name_server in name_servers:
+            f.write(f'nameserver {name_server}\n')
+        if search:
+            search_list_string = " ".join(search)
+            f.write(f'search {search_list_string}\n')
+        if sort_list:
+            sort_list_string = " ".join(sort_list)
+            f.write(f'sortlist {sort_list_string}\n')
+        if options:
+            options_string = " ".join(options)
+            f.write(f'options {options_string}\n')
+
+    # move overwritten
+    shutil.move(resolv_conf_temp, resolv_conf)
+
+
+def get_resolv_conf_name_servers(resolv_conf):
+    if not resolv_conf:
+        resolv_conf = SYSTEM_RESOLV_CONF
+
+    name_servers, _, _, _ = parse_resolv_conf(resolv_conf)
+    return name_servers
+
+
+def parse_resolv_conf(resolv_conf):
+    name_servers = []
+    search = None
+    sort_list = None
+    options = None
+    with open(resolv_conf, "r") as f:
+        lines = f.readlines()
+        for line in lines:
+            if line.startswith("#"):
+                continue
+            tokens = re.split(' |\t', line)
+            if not tokens:
+                continue
+            if tokens[0] == "nameserver":
+                if len(tokens) != 2:
+                    continue
+                name_servers.append(tokens[1])
+            elif tokens[0] == "search" or tokens[0] == "domain":
+                search = tokens[1:]
+            elif tokens[0] == "sortlist":
+                sort_list = tokens[1:]
+            elif tokens[0] == "options":
+                options = tokens[1:]
+    return name_servers, search, sort_list, options

python/cloudtik/core/config-schema.json

@@ -1696,6 +1696,11 @@
                             "type": "boolean",
                             "default": false,
                             "description": "Whether to run a server on each node for high availability."
+                        },
+                        "default_resolver": {
+                            "type": "boolean",
+                            "default": false,
+                            "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf."
                         }
                     }
                 },
@@ -1713,6 +1718,11 @@
                             "type": "boolean",
                             "default": false,
                             "description": "Whether to run a server on each node for high availability."
+                        },
+                        "default_resolver": {
+                            "type": "boolean",
+                            "default": false,
+                            "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf."
                         }
                     }
                 },
@@ -1735,6 +1745,11 @@
                             "type": "string",
                             "default": "yes",
                             "description": "The DNSSEC validation setting: yes, auto, no."
+                        },
+                        "default_resolver": {
+                            "type": "boolean",
+                            "default": false,
+                            "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf."
                         }
                     }
                 }

python/cloudtik/runtime/bind/scripts/configure.py

@@ -0,0 +1,17 @@
+import argparse
+
+from cloudtik.runtime.bind.utils import configure_upstream
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Configuring runtime.")
+    parser.add_argument('--head', action='store_true', default=False,
+                        help='Configuring for head node.')
+    args = parser.parse_args()
+
+    configure_upstream(args.head)
+
+
+if __name__ == "__main__":
+    main()

python/cloudtik/runtime/bind/scripts/configure.sh

@@ -69,6 +69,18 @@ function configure_bind() {
         echo "include \"${BIND_HOME}/conf/named.conf.consul\";" >> ${config_template_file}
     fi
 
+    SYSTEM_RESOLV_CONF="/etc/resolv.conf"
+    ORIGIN_RESOLV_CONF="${BIND_HOME}/conf/resolv.conf"
+
+    # backup the system resolv conf only once
+    if [ ! -f "${ORIGIN_RESOLV_CONF}"]; then
+        cp ${SYSTEM_RESOLV_CONF} ${ORIGIN_RESOLV_CONF}
+    fi
+
+    # python configure script will write named.conf.upstream
+    # and if necessary update the system resolv.conf
+    echo "include \"${BIND_HOME}/conf/named.conf.upstream\";" >> ${config_template_file}
+
     cp ${output_dir}/named.conf.options ${BIND_CONF_DIR}/named.conf.options
     cp ${output_dir}/named.conf.logging ${BIND_CONF_DIR}/named.conf.logging
     cp ${config_template_file} ${BIND_CONF_DIR}/named.conf

python/cloudtik/runtime/bind/scripts/services.sh

@@ -7,6 +7,11 @@ ROOT_DIR="$(dirname "$(dirname "$BIN_DIR")")"
 args=$(getopt -a -o h:: -l head:: -- "$@")
 eval set -- "${args}"
 
+USER_HOME=/home/$(whoami)
+RUNTIME_PATH=$USER_HOME/runtime
+BIND_HOME=$RUNTIME_PATH/bind
+BIND_BACKUP_RESOLV_CONF=${BIND_HOME}/conf/resolv.conf.backup
+
 # import util functions
 . "$ROOT_DIR"/common/scripts/util-functions.sh
 
@@ -16,8 +21,18 @@ set_service_command "$@"
 case "$SERVICE_COMMAND" in
 start)
     sudo service named start
+
+    if [ "${BIND_DEFAULT_RESOLVER}" == "true" ]; then
+        # update the /etc/resolv.conf
+        update_resolv_conf ${BIND_BACKUP_RESOLV_CONF} "127.0.0.1"
+    fi
     ;;
 stop)
+    if [ "${BIND_DEFAULT_RESOLVER}" == "true" ]; then
+        # restore the /etc/resolv.conf
+        restore_resolv_conf ${BIND_BACKUP_RESOLV_CONF}
+    fi
+
     sudo service named stop
     ;;
 -h|--help)

python/cloudtik/runtime/bind/utils.py

@@ -5,6 +5,7 @@
 from cloudtik.core._private.service_discovery.utils import \
     get_canonical_service_name, define_runtime_service, \
     get_service_discovery_config, SERVICE_DISCOVERY_FEATURE_DNS
+from cloudtik.core._private.util.resolv_conf import get_resolv_conf_name_servers
 from cloudtik.core._private.utils import get_runtime_config
 from cloudtik.runtime.common.service_discovery.cluster import has_runtime_in_cluster
 
@@ -18,6 +19,7 @@
 
 BIND_SERVICE_PORT_CONFIG_KEY = "port"
 BIND_DNSSEC_VALIDATION_CONFIG_KEY = "dnssec_validation"
+BIND_DEFAULT_RESOLVER_CONFIG_KEY = "default_resolver"
 
 BIND_SERVICE_NAME = BUILT_IN_RUNTIME_BIND
 BIND_SERVICE_PORT_DEFAULT = 53
@@ -58,6 +60,11 @@ def _with_runtime_environment_variables(
             cluster_runtime_config, BUILT_IN_RUNTIME_CONSUL):
         runtime_envs["BIND_CONSUL_RESOLVE"] = True
 
+    default_resolver = bind_config.get(
+        BIND_DEFAULT_RESOLVER_CONFIG_KEY, False)
+    if default_resolver:
+        runtime_envs["BIND_DEFAULT_RESOLVER"] = True
+
     return runtime_envs
 
 
@@ -74,3 +81,28 @@ def _get_runtime_services(
             features=[SERVICE_DISCOVERY_FEATURE_DNS]),
     }
     return services
+
+
+###################################
+# Calls from node when configuring
+###################################
+
+
+def configure_upstream(head):
+    conf_dir = os.path.join(
+        _get_home_dir(), "conf")
+    origin_resolv_conf = os.path.join(
+        conf_dir, "resolv.conf")
+    upstream_config_file = os.path.join(
+        conf_dir, "named.conf.upstream")
+
+    name_servers = get_resolv_conf_name_servers(
+        origin_resolv_conf)
+    with open(upstream_config_file, "w") as f:
+        f.write('zone "." {\n')
+        f.write('  type forward;\n')
+        f.write('  forwarders {\n')
+        for name_server in name_servers:
+            f.write("    {};\n".format(name_server))
+        f.write('  };\n')
+        f.write('};\n')

python/cloudtik/runtime/common/scripts/resolv-conf.py

@@ -0,0 +1,19 @@
+import argparse
+
+from cloudtik.core._private.util.resolv_conf import update_resolv_conf
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Update the /etc/resolv.conf with a list of name servers.")
+    parser.add_argument(
+        "name_servers",
+        nargs=argparse.REMAINDER,
+    )
+    args = parser.parse_args()
+    if args.name_servers:
+        update_resolv_conf(name_servers=args.name_servers)
+
+
+if __name__ == "__main__":
+    main()

python/cloudtik/runtime/common/scripts/util-functions.sh

@@ -128,3 +128,18 @@ function stop_process_by_pid_file() {
         sudo kill -15 ${MY_PID} >/dev/null 2>&1
     fi
 }
+
+function update_resolv_conf() {
+    local BACKUP_RESOLV_CONF=$1
+    cp /etc/resolv.conf ${BACKUP_RESOLV_CONF}
+    shift
+    SCRIPTS_DIR=$(dirname ${BASH_SOURCE[0]})
+    sudo python ${SCRIPTS_DIR}/resolv-conf.py "$@"
+}
+
+function restore_resolv_conf() {
+    local BACKUP_RESOLV_CONF=$1
+    if [ -f "${BACKUP_RESOLV_CONF}" ]; then
+        sudo cp ${BACKUP_RESOLV_CONF} /etc/resolv.conf
+    fi
+}

python/cloudtik/runtime/consul/utils.py

@@ -10,7 +10,7 @@
 from cloudtik.core._private.runtime_factory import BUILT_IN_RUNTIME_CONSUL
 from cloudtik.core._private.runtime_utils import get_runtime_node_type, get_runtime_node_ip, \
     get_runtime_config_from_node, RUNTIME_NODE_SEQ_ID, RUNTIME_NODE_IP, subscribe_nodes_info, sort_nodes_by_seq_id, \
-    load_and_save_json
+    load_and_save_json, get_runtime_value
 from cloudtik.core._private.service_discovery.runtime_services import get_runtime_services_by_node_type
 from cloudtik.core._private.service_discovery.utils import SERVICE_DISCOVERY_PORT, \
     SERVICE_DISCOVERY_TAGS, SERVICE_DISCOVERY_LABELS, SERVICE_DISCOVERY_LABEL_RUNTIME, \
@@ -305,12 +305,12 @@ def _get_services_of_node_type(runtime_config, node_type):
 
 
 def configure_agent(head):
-    consul_server = os.environ.get("CONSUL_SERVER")
+    consul_server = get_runtime_value("CONSUL_SERVER")
     server_mode = True if consul_server == "true" else False
     _configure_agent(server_mode, head)
 
     if server_mode:
-        quorum_join = os.environ.get(CLOUDTIK_RUNTIME_ENV_QUORUM_JOIN)
+        quorum_join = get_runtime_value(CLOUDTIK_RUNTIME_ENV_QUORUM_JOIN)
         if quorum_join == QUORUM_JOIN_STATUS_INIT:
             _update_server_config_for_join()
 
@@ -322,7 +322,7 @@ def _configure_agent(server_mode, head):
         # join list for servers
         if head:
             # for head, use its own address
-            node_ip = os.environ.get(CLOUDTIK_RUNTIME_ENV_NODE_IP)
+            node_ip = get_runtime_value(CLOUDTIK_RUNTIME_ENV_NODE_IP)
             if not node_ip:
                 raise RuntimeError("Missing node ip environment variable for the running node.")
             join_list = [node_ip]
@@ -331,19 +331,19 @@ def _configure_agent(server_mode, head):
             join_list = _get_join_list_from_nodes_info()
     else:
         # client mode, get from the CONSUL_JOIN_LIST environments
-        join_list_str = os.environ.get("CONSUL_JOIN_LIST")
+        join_list_str = get_runtime_value("CONSUL_JOIN_LIST")
         if not join_list_str:
             raise RuntimeError("Missing join list environment variable for the running node.")
         join_list = join_list_str.split(',')
 
-    cluster_name = os.environ.get(CLOUDTIK_RUNTIME_ENV_CLUSTER)
+    cluster_name = get_runtime_value(CLOUDTIK_RUNTIME_ENV_CLUSTER)
     _update_agent_config(join_list, cluster_name)
 
 
 def _get_join_list_from_nodes_info():
     nodes_info = subscribe_nodes_info()
     join_nodes = sort_nodes_by_seq_id(nodes_info)
-    head_node_ip = os.environ.get(CLOUDTIK_RUNTIME_ENV_HEAD_IP)
+    head_node_ip = get_runtime_value(CLOUDTIK_RUNTIME_ENV_HEAD_IP)
     if not head_node_ip:
         raise RuntimeError("Missing head node ip environment variable for the running node.")
 

python/cloudtik/runtime/coredns/conf/Corefile.upstream

@@ -1,7 +1,7 @@
 .:{%bind.port%} {
     bind 127.0.0.1 {%bind.ip%}
     prometheus {%bind.ip%}:9253
-    forward . /etc/resolv.conf
+    forward . {%upstream.resolv.conf%}
     log
     errors
     cache

python/cloudtik/runtime/coredns/scripts/configure.sh

@@ -50,6 +50,23 @@ function configure_coredns() {
         cp ${output_dir}/Corefile.consul ${COREDNS_CONF_DIR}/Corefile.consul
     fi
 
+    SYSTEM_RESOLV_CONF="/etc/resolv.conf"
+    ORIGIN_RESOLV_CONF="${COREDNS_HOME}/conf/resolv.conf"
+
+    # backup the system resolv conf only once
+    if [ ! -f "${ORIGIN_RESOLV_CONF}"]; then
+        cp ${SYSTEM_RESOLV_CONF} ${ORIGIN_RESOLV_CONF}
+    fi
+
+    if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then
+        UPSTREAM_RESOLV_CONF=${ORIGIN_RESOLV_CONF}
+    else
+        UPSTREAM_RESOLV_CONF=${SYSTEM_RESOLV_CONF}
+    fi
+
+    sed -i "s#{%upstream.resolv.conf%}#${UPSTREAM_RESOLV_CONF}#g" \
+      ${COREDNS_CONF_DIR}/Corefile.upstream
+
     echo "import ${COREDNS_CONF_DIR}/Corefile.upstream" >> ${config_template_file}
     cp ${output_dir}/Corefile.upstream ${COREDNS_CONF_DIR}/Corefile.upstream
 

python/cloudtik/runtime/coredns/scripts/services.sh

@@ -12,22 +12,31 @@ RUNTIME_PATH=$USER_HOME/runtime
 COREDNS_HOME=$RUNTIME_PATH/coredns
 COREDNS_CONFIG_FILE=${COREDNS_HOME}/conf/Corefile
 COREDNS_PID_FILE=${COREDNS_HOME}/coredns.pid
+COREDNS_BACKUP_RESOLV_CONF=${COREDNS_HOME}/conf/resolv.conf.backup
 
 # import util functions
 . "$ROOT_DIR"/common/scripts/util-functions.sh
 
 set_head_option "$@"
 set_service_command "$@"
 
-
 case "$SERVICE_COMMAND" in
 start)
     sudo nohup ${COREDNS_HOME}/coredns \
       -conf ${COREDNS_CONFIG_FILE} \
       -pidfile ${COREDNS_PID_FILE} \
       >${COREDNS_HOME}/logs/coredns.log 2>&1 &
+
+    if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then
+        # update the /etc/resolv.conf
+        update_resolv_conf ${COREDNS_BACKUP_RESOLV_CONF} "127.0.0.1"
+    fi
     ;;
 stop)
+    if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then
+        # restore the /etc/resolv.conf
+        restore_resolv_conf ${COREDNS_BACKUP_RESOLV_CONF}
+    fi
     stop_process_by_pid_file "${COREDNS_PID_FILE}"
     ;;
 -h|--help)