diff --git a/python/cloudtik/core/_private/util/resolv_conf.py b/python/cloudtik/core/_private/util/resolv_conf.py new file mode 100644 index 000000000..83bc69f99 --- /dev/null +++ b/python/cloudtik/core/_private/util/resolv_conf.py @@ -0,0 +1,65 @@ +import re +import shutil +import tempfile + +SYSTEM_RESOLV_CONF = "/etc/resolv.conf" + + +def update_resolv_conf(name_servers, resolv_conf=None): + if not resolv_conf: + resolv_conf = SYSTEM_RESOLV_CONF + + old_name_servers, search, sort_list, options = parse_resolv_conf( + resolv_conf) + + # write to temp file first + resolv_conf_temp = tempfile.mktemp(prefix=f"resolv.conf_") + with open(resolv_conf_temp, "w") as f: + for name_server in name_servers: + f.write(f'nameserver {name_server}\n') + if search: + search_list_string = " ".join(search) + f.write(f'search {search_list_string}\n') + if sort_list: + sort_list_string = " ".join(sort_list) + f.write(f'sortlist {sort_list_string}\n') + if options: + options_string = " ".join(options) + f.write(f'options {options_string}\n') + + # move overwritten + shutil.move(resolv_conf_temp, resolv_conf) + + +def get_resolv_conf_name_servers(resolv_conf): + if not resolv_conf: + resolv_conf = SYSTEM_RESOLV_CONF + + name_servers, _, _, _ = parse_resolv_conf(resolv_conf) + return name_servers + + +def parse_resolv_conf(resolv_conf): + name_servers = [] + search = None + sort_list = None + options = None + with open(resolv_conf, "r") as f: + lines = f.readlines() + for line in lines: + if line.startswith("#"): + continue + tokens = re.split(' |\t', line) + if not tokens: + continue + if tokens[0] == "nameserver": + if len(tokens) != 2: + continue + name_servers.append(tokens[1]) + elif tokens[0] == "search" or tokens[0] == "domain": + search = tokens[1:] + elif tokens[0] == "sortlist": + sort_list = tokens[1:] + elif tokens[0] == "options": + options = tokens[1:] + return name_servers, search, sort_list, options diff --git a/python/cloudtik/core/config-schema.json b/python/cloudtik/core/config-schema.json index 3557dba72..a9e211a7f 100644 --- a/python/cloudtik/core/config-schema.json +++ b/python/cloudtik/core/config-schema.json @@ -1696,6 +1696,11 @@ "type": "boolean", "default": false, "description": "Whether to run a server on each node for high availability." + }, + "default_resolver": { + "type": "boolean", + "default": false, + "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf." } } }, @@ -1713,6 +1718,11 @@ "type": "boolean", "default": false, "description": "Whether to run a server on each node for high availability." + }, + "default_resolver": { + "type": "boolean", + "default": false, + "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf." } } }, @@ -1735,6 +1745,11 @@ "type": "string", "default": "yes", "description": "The DNSSEC validation setting: yes, auto, no." + }, + "default_resolver": { + "type": "boolean", + "default": false, + "description": "Whether to set this DNS server as the system default resolver in /etc/resolv.conf." } } } diff --git a/python/cloudtik/runtime/bind/scripts/configure.py b/python/cloudtik/runtime/bind/scripts/configure.py new file mode 100644 index 000000000..d2ea4e9e5 --- /dev/null +++ b/python/cloudtik/runtime/bind/scripts/configure.py @@ -0,0 +1,17 @@ +import argparse + +from cloudtik.runtime.bind.utils import configure_upstream + + +def main(): + parser = argparse.ArgumentParser( + description="Configuring runtime.") + parser.add_argument('--head', action='store_true', default=False, + help='Configuring for head node.') + args = parser.parse_args() + + configure_upstream(args.head) + + +if __name__ == "__main__": + main() diff --git a/python/cloudtik/runtime/bind/scripts/configure.sh b/python/cloudtik/runtime/bind/scripts/configure.sh index 97ffdd9d8..20dbd1c65 100644 --- a/python/cloudtik/runtime/bind/scripts/configure.sh +++ b/python/cloudtik/runtime/bind/scripts/configure.sh @@ -69,6 +69,18 @@ function configure_bind() { echo "include \"${BIND_HOME}/conf/named.conf.consul\";" >> ${config_template_file} fi + SYSTEM_RESOLV_CONF="/etc/resolv.conf" + ORIGIN_RESOLV_CONF="${BIND_HOME}/conf/resolv.conf" + + # backup the system resolv conf only once + if [ ! -f "${ORIGIN_RESOLV_CONF}"]; then + cp ${SYSTEM_RESOLV_CONF} ${ORIGIN_RESOLV_CONF} + fi + + # python configure script will write named.conf.upstream + # and if necessary update the system resolv.conf + echo "include \"${BIND_HOME}/conf/named.conf.upstream\";" >> ${config_template_file} + cp ${output_dir}/named.conf.options ${BIND_CONF_DIR}/named.conf.options cp ${output_dir}/named.conf.logging ${BIND_CONF_DIR}/named.conf.logging cp ${config_template_file} ${BIND_CONF_DIR}/named.conf diff --git a/python/cloudtik/runtime/bind/scripts/services.sh b/python/cloudtik/runtime/bind/scripts/services.sh index 3ecec1b9d..c168bd504 100644 --- a/python/cloudtik/runtime/bind/scripts/services.sh +++ b/python/cloudtik/runtime/bind/scripts/services.sh @@ -7,6 +7,11 @@ ROOT_DIR="$(dirname "$(dirname "$BIN_DIR")")" args=$(getopt -a -o h:: -l head:: -- "$@") eval set -- "${args}" +USER_HOME=/home/$(whoami) +RUNTIME_PATH=$USER_HOME/runtime +BIND_HOME=$RUNTIME_PATH/bind +BIND_BACKUP_RESOLV_CONF=${BIND_HOME}/conf/resolv.conf.backup + # import util functions . "$ROOT_DIR"/common/scripts/util-functions.sh @@ -16,8 +21,18 @@ set_service_command "$@" case "$SERVICE_COMMAND" in start) sudo service named start + + if [ "${BIND_DEFAULT_RESOLVER}" == "true" ]; then + # update the /etc/resolv.conf + update_resolv_conf ${BIND_BACKUP_RESOLV_CONF} "127.0.0.1" + fi ;; stop) + if [ "${BIND_DEFAULT_RESOLVER}" == "true" ]; then + # restore the /etc/resolv.conf + restore_resolv_conf ${BIND_BACKUP_RESOLV_CONF} + fi + sudo service named stop ;; -h|--help) diff --git a/python/cloudtik/runtime/bind/utils.py b/python/cloudtik/runtime/bind/utils.py index e7d71ed85..001472e9e 100644 --- a/python/cloudtik/runtime/bind/utils.py +++ b/python/cloudtik/runtime/bind/utils.py @@ -5,6 +5,7 @@ from cloudtik.core._private.service_discovery.utils import \ get_canonical_service_name, define_runtime_service, \ get_service_discovery_config, SERVICE_DISCOVERY_FEATURE_DNS +from cloudtik.core._private.util.resolv_conf import get_resolv_conf_name_servers from cloudtik.core._private.utils import get_runtime_config from cloudtik.runtime.common.service_discovery.cluster import has_runtime_in_cluster @@ -18,6 +19,7 @@ BIND_SERVICE_PORT_CONFIG_KEY = "port" BIND_DNSSEC_VALIDATION_CONFIG_KEY = "dnssec_validation" +BIND_DEFAULT_RESOLVER_CONFIG_KEY = "default_resolver" BIND_SERVICE_NAME = BUILT_IN_RUNTIME_BIND BIND_SERVICE_PORT_DEFAULT = 53 @@ -58,6 +60,11 @@ def _with_runtime_environment_variables( cluster_runtime_config, BUILT_IN_RUNTIME_CONSUL): runtime_envs["BIND_CONSUL_RESOLVE"] = True + default_resolver = bind_config.get( + BIND_DEFAULT_RESOLVER_CONFIG_KEY, False) + if default_resolver: + runtime_envs["BIND_DEFAULT_RESOLVER"] = True + return runtime_envs @@ -74,3 +81,28 @@ def _get_runtime_services( features=[SERVICE_DISCOVERY_FEATURE_DNS]), } return services + + +################################### +# Calls from node when configuring +################################### + + +def configure_upstream(head): + conf_dir = os.path.join( + _get_home_dir(), "conf") + origin_resolv_conf = os.path.join( + conf_dir, "resolv.conf") + upstream_config_file = os.path.join( + conf_dir, "named.conf.upstream") + + name_servers = get_resolv_conf_name_servers( + origin_resolv_conf) + with open(upstream_config_file, "w") as f: + f.write('zone "." {\n') + f.write(' type forward;\n') + f.write(' forwarders {\n') + for name_server in name_servers: + f.write(" {};\n".format(name_server)) + f.write(' };\n') + f.write('};\n') diff --git a/python/cloudtik/runtime/common/scripts/resolv-conf.py b/python/cloudtik/runtime/common/scripts/resolv-conf.py new file mode 100644 index 000000000..42be5546c --- /dev/null +++ b/python/cloudtik/runtime/common/scripts/resolv-conf.py @@ -0,0 +1,19 @@ +import argparse + +from cloudtik.core._private.util.resolv_conf import update_resolv_conf + + +def main(): + parser = argparse.ArgumentParser( + description="Update the /etc/resolv.conf with a list of name servers.") + parser.add_argument( + "name_servers", + nargs=argparse.REMAINDER, + ) + args = parser.parse_args() + if args.name_servers: + update_resolv_conf(name_servers=args.name_servers) + + +if __name__ == "__main__": + main() diff --git a/python/cloudtik/runtime/common/scripts/util-functions.sh b/python/cloudtik/runtime/common/scripts/util-functions.sh index 61a302fd4..44715f5a2 100644 --- a/python/cloudtik/runtime/common/scripts/util-functions.sh +++ b/python/cloudtik/runtime/common/scripts/util-functions.sh @@ -128,3 +128,18 @@ function stop_process_by_pid_file() { sudo kill -15 ${MY_PID} >/dev/null 2>&1 fi } + +function update_resolv_conf() { + local BACKUP_RESOLV_CONF=$1 + cp /etc/resolv.conf ${BACKUP_RESOLV_CONF} + shift + SCRIPTS_DIR=$(dirname ${BASH_SOURCE[0]}) + sudo python ${SCRIPTS_DIR}/resolv-conf.py "$@" +} + +function restore_resolv_conf() { + local BACKUP_RESOLV_CONF=$1 + if [ -f "${BACKUP_RESOLV_CONF}" ]; then + sudo cp ${BACKUP_RESOLV_CONF} /etc/resolv.conf + fi +} diff --git a/python/cloudtik/runtime/consul/utils.py b/python/cloudtik/runtime/consul/utils.py index a2497cc5f..b563268fa 100644 --- a/python/cloudtik/runtime/consul/utils.py +++ b/python/cloudtik/runtime/consul/utils.py @@ -10,7 +10,7 @@ from cloudtik.core._private.runtime_factory import BUILT_IN_RUNTIME_CONSUL from cloudtik.core._private.runtime_utils import get_runtime_node_type, get_runtime_node_ip, \ get_runtime_config_from_node, RUNTIME_NODE_SEQ_ID, RUNTIME_NODE_IP, subscribe_nodes_info, sort_nodes_by_seq_id, \ - load_and_save_json + load_and_save_json, get_runtime_value from cloudtik.core._private.service_discovery.runtime_services import get_runtime_services_by_node_type from cloudtik.core._private.service_discovery.utils import SERVICE_DISCOVERY_PORT, \ SERVICE_DISCOVERY_TAGS, SERVICE_DISCOVERY_LABELS, SERVICE_DISCOVERY_LABEL_RUNTIME, \ @@ -305,12 +305,12 @@ def _get_services_of_node_type(runtime_config, node_type): def configure_agent(head): - consul_server = os.environ.get("CONSUL_SERVER") + consul_server = get_runtime_value("CONSUL_SERVER") server_mode = True if consul_server == "true" else False _configure_agent(server_mode, head) if server_mode: - quorum_join = os.environ.get(CLOUDTIK_RUNTIME_ENV_QUORUM_JOIN) + quorum_join = get_runtime_value(CLOUDTIK_RUNTIME_ENV_QUORUM_JOIN) if quorum_join == QUORUM_JOIN_STATUS_INIT: _update_server_config_for_join() @@ -322,7 +322,7 @@ def _configure_agent(server_mode, head): # join list for servers if head: # for head, use its own address - node_ip = os.environ.get(CLOUDTIK_RUNTIME_ENV_NODE_IP) + node_ip = get_runtime_value(CLOUDTIK_RUNTIME_ENV_NODE_IP) if not node_ip: raise RuntimeError("Missing node ip environment variable for the running node.") join_list = [node_ip] @@ -331,19 +331,19 @@ def _configure_agent(server_mode, head): join_list = _get_join_list_from_nodes_info() else: # client mode, get from the CONSUL_JOIN_LIST environments - join_list_str = os.environ.get("CONSUL_JOIN_LIST") + join_list_str = get_runtime_value("CONSUL_JOIN_LIST") if not join_list_str: raise RuntimeError("Missing join list environment variable for the running node.") join_list = join_list_str.split(',') - cluster_name = os.environ.get(CLOUDTIK_RUNTIME_ENV_CLUSTER) + cluster_name = get_runtime_value(CLOUDTIK_RUNTIME_ENV_CLUSTER) _update_agent_config(join_list, cluster_name) def _get_join_list_from_nodes_info(): nodes_info = subscribe_nodes_info() join_nodes = sort_nodes_by_seq_id(nodes_info) - head_node_ip = os.environ.get(CLOUDTIK_RUNTIME_ENV_HEAD_IP) + head_node_ip = get_runtime_value(CLOUDTIK_RUNTIME_ENV_HEAD_IP) if not head_node_ip: raise RuntimeError("Missing head node ip environment variable for the running node.") diff --git a/python/cloudtik/runtime/coredns/conf/Corefile.upstream b/python/cloudtik/runtime/coredns/conf/Corefile.upstream index 4d3d616d8..54dd0d1a0 100644 --- a/python/cloudtik/runtime/coredns/conf/Corefile.upstream +++ b/python/cloudtik/runtime/coredns/conf/Corefile.upstream @@ -1,7 +1,7 @@ .:{%bind.port%} { bind 127.0.0.1 {%bind.ip%} prometheus {%bind.ip%}:9253 - forward . /etc/resolv.conf + forward . {%upstream.resolv.conf%} log errors cache diff --git a/python/cloudtik/runtime/coredns/scripts/configure.sh b/python/cloudtik/runtime/coredns/scripts/configure.sh index 49c4f09d9..384b45d6b 100644 --- a/python/cloudtik/runtime/coredns/scripts/configure.sh +++ b/python/cloudtik/runtime/coredns/scripts/configure.sh @@ -50,6 +50,23 @@ function configure_coredns() { cp ${output_dir}/Corefile.consul ${COREDNS_CONF_DIR}/Corefile.consul fi + SYSTEM_RESOLV_CONF="/etc/resolv.conf" + ORIGIN_RESOLV_CONF="${COREDNS_HOME}/conf/resolv.conf" + + # backup the system resolv conf only once + if [ ! -f "${ORIGIN_RESOLV_CONF}"]; then + cp ${SYSTEM_RESOLV_CONF} ${ORIGIN_RESOLV_CONF} + fi + + if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then + UPSTREAM_RESOLV_CONF=${ORIGIN_RESOLV_CONF} + else + UPSTREAM_RESOLV_CONF=${SYSTEM_RESOLV_CONF} + fi + + sed -i "s#{%upstream.resolv.conf%}#${UPSTREAM_RESOLV_CONF}#g" \ + ${COREDNS_CONF_DIR}/Corefile.upstream + echo "import ${COREDNS_CONF_DIR}/Corefile.upstream" >> ${config_template_file} cp ${output_dir}/Corefile.upstream ${COREDNS_CONF_DIR}/Corefile.upstream diff --git a/python/cloudtik/runtime/coredns/scripts/services.sh b/python/cloudtik/runtime/coredns/scripts/services.sh index 2a02b4fea..f80258c42 100644 --- a/python/cloudtik/runtime/coredns/scripts/services.sh +++ b/python/cloudtik/runtime/coredns/scripts/services.sh @@ -12,6 +12,7 @@ RUNTIME_PATH=$USER_HOME/runtime COREDNS_HOME=$RUNTIME_PATH/coredns COREDNS_CONFIG_FILE=${COREDNS_HOME}/conf/Corefile COREDNS_PID_FILE=${COREDNS_HOME}/coredns.pid +COREDNS_BACKUP_RESOLV_CONF=${COREDNS_HOME}/conf/resolv.conf.backup # import util functions . "$ROOT_DIR"/common/scripts/util-functions.sh @@ -19,15 +20,23 @@ COREDNS_PID_FILE=${COREDNS_HOME}/coredns.pid set_head_option "$@" set_service_command "$@" - case "$SERVICE_COMMAND" in start) sudo nohup ${COREDNS_HOME}/coredns \ -conf ${COREDNS_CONFIG_FILE} \ -pidfile ${COREDNS_PID_FILE} \ >${COREDNS_HOME}/logs/coredns.log 2>&1 & + + if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then + # update the /etc/resolv.conf + update_resolv_conf ${COREDNS_BACKUP_RESOLV_CONF} "127.0.0.1" + fi ;; stop) + if [ "${COREDNS_DEFAULT_RESOLVER}" == "true" ]; then + # restore the /etc/resolv.conf + restore_resolv_conf ${COREDNS_BACKUP_RESOLV_CONF} + fi stop_process_by_pid_file "${COREDNS_PID_FILE}" ;; -h|--help) diff --git a/python/cloudtik/runtime/coredns/utils.py b/python/cloudtik/runtime/coredns/utils.py index 0687e14b2..8cb07db93 100644 --- a/python/cloudtik/runtime/coredns/utils.py +++ b/python/cloudtik/runtime/coredns/utils.py @@ -17,6 +17,7 @@ ] COREDNS_SERVICE_PORT_CONFIG_KEY = "port" +COREDNS_DEFAULT_RESOLVER_CONFIG_KEY = "default_resolver" COREDNS_SERVICE_NAME = BUILT_IN_RUNTIME_COREDNS COREDNS_METRICS_SERVICE_NAME = BUILT_IN_RUNTIME_COREDNS + "-metrics" @@ -55,6 +56,11 @@ def _with_runtime_environment_variables( cluster_runtime_config, BUILT_IN_RUNTIME_CONSUL): runtime_envs["COREDNS_CONSUL_RESOLVE"] = True + default_resolver = coredns_config.get( + COREDNS_DEFAULT_RESOLVER_CONFIG_KEY, False) + if default_resolver: + runtime_envs["COREDNS_DEFAULT_RESOLVER"] = True + return runtime_envs diff --git a/python/cloudtik/runtime/dnsmasq/conf/dnsmasq.conf b/python/cloudtik/runtime/dnsmasq/conf/dnsmasq.conf index 275531b6f..b1bc7f215 100644 --- a/python/cloudtik/runtime/dnsmasq/conf/dnsmasq.conf +++ b/python/cloudtik/runtime/dnsmasq/conf/dnsmasq.conf @@ -16,7 +16,7 @@ listen-address=127.0.0.1,{%listen.address%} # the first file name specified overrides the default, subsequent # ones add to the list. This is only allowed when polling; the file # with the currently latest modification time is the one used. -#resolv-file= +resolv-file={%upstream.resolv.conf%} # By default, dnsmasq will send queries to any of the upstream # servers it knows about and tries to favour servers to are known diff --git a/python/cloudtik/runtime/dnsmasq/scripts/configure.sh b/python/cloudtik/runtime/dnsmasq/scripts/configure.sh index 80b0e0787..f5f797666 100644 --- a/python/cloudtik/runtime/dnsmasq/scripts/configure.sh +++ b/python/cloudtik/runtime/dnsmasq/scripts/configure.sh @@ -49,10 +49,28 @@ function configure_dnsmasq() { # dnsmasq will use /etc/resolv.conf for upstream. # TODO: if we want to use this DNS server as the system default, we need: - # 1. copy the /etc/resolv.conf to new file (how do we know this is the original) - # 2. direct dnsmasq to use the new copy as upstream + # 1. copy the /etc/resolv.conf to a backup file if backup file not exists + # 2. direct dnsmasq to use the backup copy as upstream # 3. modify /etc/resolve.conf to use dnsmasq as resolver + SYSTEM_RESOLV_CONF="/etc/resolv.conf" + ORIGIN_RESOLV_CONF="${DNSMASQ_HOME}/conf/resolv.conf" + + # backup the system resolv conf only once + if [ ! -f "${ORIGIN_RESOLV_CONF}"]; then + cp ${SYSTEM_RESOLV_CONF} ${ORIGIN_RESOLV_CONF} + fi + + if [ "${DNSMASQ_DEFAULT_RESOLVER}" == "true" ]; then + UPSTREAM_RESOLV_CONF=${ORIGIN_RESOLV_CONF} + else + UPSTREAM_RESOLV_CONF=${SYSTEM_RESOLV_CONF} + fi + + sed -i "s#{%upstream.resolv.conf%}#${UPSTREAM_RESOLV_CONF}#g" \ + ${config_template_file} + # python configure script will update the system resolv.conf + cp ${config_template_file} ${DNSMASQ_CONF_INCLUDE_DIR}/dnsmasq.conf # generate additional name server records for specific (service discovery) domain diff --git a/python/cloudtik/runtime/dnsmasq/scripts/services.sh b/python/cloudtik/runtime/dnsmasq/scripts/services.sh index 03c049963..18cc92ee2 100644 --- a/python/cloudtik/runtime/dnsmasq/scripts/services.sh +++ b/python/cloudtik/runtime/dnsmasq/scripts/services.sh @@ -7,6 +7,11 @@ ROOT_DIR="$(dirname "$(dirname "$BIN_DIR")")" args=$(getopt -a -o h:: -l head:: -- "$@") eval set -- "${args}" +USER_HOME=/home/$(whoami) +RUNTIME_PATH=$USER_HOME/runtime +DNSMASQ_HOME=$RUNTIME_PATH/dnsmasq +DNSMASQ_BACKUP_RESOLV_CONF=${DNSMASQ_HOME}/conf/resolv.conf.backup + # import util functions . "$ROOT_DIR"/common/scripts/util-functions.sh @@ -16,8 +21,18 @@ set_service_command "$@" case "$SERVICE_COMMAND" in start) sudo service dnsmasq start + + if [ "${DNSMASQ_DEFAULT_RESOLVER}" == "true" ]; then + # update the /etc/resolv.conf + update_resolv_conf ${DNSMASQ_BACKUP_RESOLV_CONF} "127.0.0.1" + fi ;; stop) + if [ "${DNSMASQ_DEFAULT_RESOLVER}" == "true" ]; then + # restore the /etc/resolv.conf + restore_resolv_conf ${DNSMASQ_BACKUP_RESOLV_CONF} + fi + sudo service dnsmasq stop ;; -h|--help) diff --git a/python/cloudtik/runtime/dnsmasq/utils.py b/python/cloudtik/runtime/dnsmasq/utils.py index d8bca54b2..e743e2770 100644 --- a/python/cloudtik/runtime/dnsmasq/utils.py +++ b/python/cloudtik/runtime/dnsmasq/utils.py @@ -17,6 +17,7 @@ ] DNSMASQ_SERVICE_PORT_CONFIG_KEY = "port" +DNSMASQ_DEFAULT_RESOLVER_CONFIG_KEY = "default_resolver" DNSMASQ_SERVICE_NAME = BUILT_IN_RUNTIME_DNSMASQ DNSMASQ_SERVICE_PORT_DEFAULT = 53 @@ -53,6 +54,11 @@ def _with_runtime_environment_variables( cluster_runtime_config, BUILT_IN_RUNTIME_CONSUL): runtime_envs["DNSMASQ_CONSUL_RESOLVE"] = True + default_resolver = dnsmasq_config.get( + DNSMASQ_DEFAULT_RESOLVER_CONFIG_KEY, False) + if default_resolver: + runtime_envs["DNSMASQ_DEFAULT_RESOLVER"] = True + return runtime_envs