#!/usr/bin/env python3
from __future__ import unicode_literals

import argparse
import atexit
from dataclasses import dataclass
import json
import logging
import multiprocessing
import os
import re
import resource
import shutil
import subprocess
import sys
import time
import traceback
import uuid
import socket
import tarfile
import operator
import string
import glob
import tarfile
import ipaddress
from datetime import datetime, timedelta
from signal import SIGABRT, SIGINT, SIGKILL, SIGTERM, SIG_DFL, SIG_IGN, signal
from threading import Thread, Event

# Version-dependent imports
PY_VERSION = sys.version_info[0]
if PY_VERSION < 3:
    import Queue as queue
    from urllib2 import Request, urlopen, URLError, HTTPError
    from urllib import urlencode
    from random import SystemRandom
    _sysrand = SystemRandom()
    PASSWORD_GENNERATOR = _sysrand.choice
else:
    import queue
    from urllib.request import Request, urlopen
    from urllib.error import URLError, HTTPError
    from urllib.parse import urlencode, urlparse
    import secrets
    PASSWORD_GENNERATOR = secrets.choice

"""
Run `yugabyted` to start a single-node YugabyteDB process. If no options are specified,
`yugabyted` will assume the following default directory tree:

yugabyte
+-- var
    |
    +-- conf
        |   +-- yugabyted.conf
     +-- logs
         |   +-- master & tserver & yugaware
     +-- data
+-- bin
|   |   +-- yugabyted
|   |   +-- yb-master
|   |   +-- yb-tserver
|   |   +-- ...
+-- ui
|   |   +-- bin...
|   |   +-- ...
"""
# OS constants
OS_DETAILS = os.uname()
OS_NAME = OS_DETAILS[0]

# Script constants.
SCRIPT_NAME = os.path.basename(__file__)
YUGABYTE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
YUGABYTE_JENKINS_BUILD_DIR = os.getenv("BUILD_ROOT",
                                       os.path.dirname(
                                           os.path.dirname(os.path.realpath(__file__))))
TRUE_CHOICES = ["true", "True", "t", "T", "yes", "Yes", "y", "Y", "1"]
FALSE_CHOICES = ["false", "False", "f", "F", "no", "No", "n", "N", "0"]
BOOL_CHOICES = TRUE_CHOICES + FALSE_CHOICES
FAULT_TOLERANCE_CHOICES = ["zone", "region", "cloud"]
START_FAULT_TOLERANCE_CHOICES = ["none", "zone", "region", "cloud"]
SLACK_LINK = "https://www.yugabyte.com/slack"
COMMUNITY_REWARDS_LINK = "https://www.yugabyte.com/community-rewards/"
HELP_LINK = "https://docs.yugabyte.com/latest/faq/"
YUGABYTED_LINK = "https://docs.yugabyte.com/preview/reference/configuration/yugabyted/"
YUGABYTED_START = "https://docs.yugabyte.com/preview/reference/configuration/yugabyted/#start"
GENERATE_SERVER_CERTS = "https://docs.yugabyte.com/preview/secure/tls-encryption/\
server-certificates/#create-the-server-certificates"
YBC_DOWNLOAD_LINK = "https://s3.us-west-2.amazonaws.com/downloads.yugabyte.com/ybc/" + \
"2.1.0.0-b9/ybc-2.1.0.0-b9-linux-x86_64.tar.gz"
DEFAULT_DEMO_DATABASE = "northwind"
DEFAULT_FAULT_TOLERANCE = "zone"
DEFAULT_START_FAULT_TOLERANCE = "none"
DEAFULT_RETENTION_PERIOD = 7
CALLHOME_TYPE = "yugabyted"
CALLHOME_EVENT_START = "start"
CALLHOME_EVENT_HEARTBEAT_STATUS = "heartbeat_status"
CALLHOME_EVENT_BACKUP = "backup"
CALLHOME_EVENT_RESTORE = "restore"
CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT = "xcluster_create_checkpoint"
CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT = "xcluster_add_to_checkpoint"
CALLHOME_EVENT_XCLUSTER_SET_UP = "xcluster_set_up"
CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION = "xcluster_add_to_replication"
CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION = "xcluster_delete_replication"
CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION = \
    "xcluster_remove_database_from_replication"
CALLHOME_EVENT_PITR_ENABLE = "pitr_enable"
CALLHOME_EVENT_PITR_DISABLE = "pitr_disable"
CALLHOME_EVENT_DEMO_CONNECT = "demo_connect"
CALLHOME_EVENT_UPGRADE_YSQL_CATALOG = "upgrade_ysql_catalog"
CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION = "upgrade_finalize_new_version"
CALLHOME_EVENT_UPGRADE_CHECK_VERSION_COMPATIBILITY = \
    "upgrade_check_version_compatibility"
SAMPLE_DATA_LINKS = {
    "retail": "https://docs.yugabyte.com/latest/quick-start/explore-ysql/",
    "chinook": "https://docs.yugabyte.com/latest/sample-data/chinook/",
    "sports": "https://docs.yugabyte.com/latest/sample-data/sportsdb/",
    "northwind": "https://docs.yugabyte.com/latest/sample-data/northwind/"
}
SYSTEM_NAMESPACES = [
    "system_schema", "system_auth", "system",
    "system_platform", "template0", "template1"
]
EXIT_SIGNALS = (SIGABRT, SIGINT, SIGTERM)

MAX_PROC = {
    'Linux' : 12000,
    'Darwin' : 2500,
}
PREREQS_ERROR_MSGS = {
    'open_files' :'open files ulimits value set low. Please set soft and hard limits to 1048576.',
    'max_user_processes' :'max user processes ulimits value set low.' \
        ' Please set soft and hard limits to {}.'.format(MAX_PROC[OS_NAME]),
    'transparent_hugepages' :'Transparent hugepages disabled. Please enable transparent_hugepages.',
    'ntp/chrony' :'ntp/chrony package is missing for clock synchronization. For centos 7, ' +
        'we recommend installing either ntp or chrony package and for centos 8, ' +
        'we recommend installing chrony package.',
    'mandatory_ports': 'YugabyteDB processes cannot start as the default port(s) {} already' \
            ' in use. Please free the port(s) and rerun the {} command.',
    'yugabyted_ui_port': 'Yugabyted UI cannot start as the default port {} is already in use.' \
            ' Please free the port and restart with --ui=true.',
    'ysql_metric_port': 'YSQL metrics port {} is already in use. For accessing the YSQL metrics,' \
            ' please free the port and restart the node.',
    'ycql_metric_port': 'YCQL metrics port {} is already in use. For accessing the YCQL metrics,' \
            ' please free the port and restart the node.',
    'clockbound_fail': 'Failed to validate system configuration for clockbound. Please run ' \
        'bin/configure_clockbound.sh script to install and configure clockbound.',
    'clockbound': 'Clockbound is recommended on AWS/Azure/GCP clusters.' \
        ' It can reduce read restart errors significantly in concurrent workloads. Please run ' \
        'bin/configure_clockbound.sh script to install and configure clockbound.',
    'python_version': 'Python 3.11+ is required for the Distributed RAG service. ' \
        'Please install Python 3.11 or higher.',
}
QUICK_START_LINKS = {
    'mac' : 'https://docs.yugabyte.com/preview/quick-start/',
    'linux' : 'https://docs.yugabyte.com/preview/quick-start/linux/',
}
CONFIG_LINK = "https://docs.yugabyte.com/latest/deploy/manual-deployment/system-config"
DEFAULT_PORTS_LINK = "https://docs.yugabyte.com/preview/reference/configuration/default-ports/"

# Help Message Constants
PREFIX = {
    'yugabyted' : "YugabyteDB command-line interface for creating" +
                    " and configuring YugabyteDB cluster.",
    'start' : "Install YugabyteDB and start a single node cluster.\n\n" +
                "Use --join flag to join other nodes that are part of the same cluster.",
    'stop' : "",
    'destroy' : "",
    'status' : "",
    'version' : "",
    'collect_logs' : "",
    'configure' : "",
    'configure data_placement': "",
    'configure encrypt_at_rest': "",
    'configure admin_operation': "",
    'configure point_in_time_recovery': "",
    'configure_read_replica' : "",
    'configure_read_replica new': "",
    'configure_read_replica modify': "",
    'configure_read_replica delete': "",
    'upgrade' : "Upgrade the YugabyteDB node.",
    'upgrade ysql_catalog' : "",
    'upgrade finalize_new_version'  : "",
    "upgrade check_version_compatibility" : "",
    'xcluster' : "Setup and manage xCluster Replication.",
    'xcluster create_checkpoint': "",
    'xcluster add_to_checkpoint': "",
    'xcluster set_up': "",
    'xcluster add_to_replication': "",
    'xcluster status': "",
    'xcluster delete_replication': "",
    'xcluster remove_database_from_replication': "",
    'backup': "",
    'restore': "",
    'connect' : "",
    'connect ycql' : "",
    'connect ysql' : "",
    'demo' : "",
    'demo connect' : "",
    'demo destroy' : "",
    'cert' : "",
    'cert generate_server_certs' : "",
    'finalize_upgrade' : "",
}

USAGE = {
    'yugabyted' : "yugabyted [command] [flags]",
    'start' : "yugabyted start [flags]",
    'stop' : "yugabyted stop [flags]",
    'destroy' : "yugabyted destroy [flags]",
    'status' : "yugabyted status [flags]",
    'version' : "yugabyted version [flags]",
    'collect_logs' : "yugabyted collect_logs [flags]",
    'configure' : "yugabyted configure [command] [flags]",
    'configure data_placement': "yugabyted configure data_placement [flags]",
    'configure encrypt_at_rest': "yugabyted configure encrypt_at_rest [flags]",
    'configure admin_operation' : "yugabyted configure admin_operation [flags]",
    'configure point_in_time_recovery' : "yugabyted configure point_in_time_recovery [flags]",
    'configure_read_replica' : "yugabyted configure_read_replica [command] [flags]",
    'configure_read_replica new': "yugabyted configure_read_replica new [flags]",
    'configure_read_replica modify': "yugabyted configure_read_replica modify [flags]",
    'configure_read_replica delete' : "yugabyted configure_read_replica delete",
    'upgrade' : "yugabyted upgrade [command] [flags]",
    'upgrade ysql_catalog' : "yugabyted upgrade ysql_catalog [flags]",
    'upgrade finalize_new_version' : "yugabyted upgrade finalize [flags]",
    'upgrade check_version_compatibility' : "yugabyted upgrade check_version_compatibility [flags]",
    'xcluster' : "yugabyted xcluster [command] [flags]",
    'xcluster create_checkpoint': "yugabyted xcluster create_checkpoint [flags]",
    'xcluster add_to_checkpoint': "yugabyted xcluster add_to_checkpoint [flags]",
    'xcluster set_up': "yugabyted xcluster set_up [flags]",
    'xcluster add_to_replication': "yugabyted xcluster add_to_replication [flags]",
    'xcluster status': "yugabyted xcluster status [flags]",
    'xcluster delete_replication': "yugabyted xcluster delete_replication [flags]",
    'xcluster remove_database_from_replication': "yugabyted xcluster " +
                                                    "remove_database_from_replication [flags]",
    'backup' : "yugabyted backup [flags]",
    'restore' : "yugabyted restore [flags]",
    'connect' : "yugabyted connect [command] [flags]",
    'connect ycql' : "yugabyted connect ycql [flags]",
    'connect ysql' : "yugabyted connect ysql [flags]",
    'demo' : "yugabyted demo [command] [flags]",
    'demo connect' : "yugabyted demo connect [flags]",
    'demo destroy' : "yugabyted demo destroy [flags]",
    'cert' : "yugabyted cert [command] [flags]",
    'cert generate_server_certs' : "yugabyted cert generate_server_certs [flag]",
    'finalize_upgrade': "yugabyted finalize_upgrade [flags]",
}

EXAMPLE = {
    'start' : "# Create a single-node local cluster:\n" +
              "yugabyted start\n\n"+
              "# Create a single-node locally and join other nodes " +
              "that are part of the same cluster:\n" +
              "yugabyted start --join=host:port,[host:port]\n\n" +
              "# Create a secure cluster:\n" +
              "yugabyted start --secure --certs_dir=<path_to_certs_dir>\n\n" +
              "# Use precise clocks:\n" +
              "yugabyted start --enhance_time_sync_via_clockbound\n\n",
    'stop' : "",
    'destroy' : "",
    'status' : "",
    'version' : "",
    'collect_logs' : "",
    'cert' : "# Create node sever certificates:\n" +
             "yugabyted cert generate_server_certs --hostnames=<comma_seperated_hostnames>\n\n",
    'backup' : "# Backup a YSQL Database into AWS S3 Bucket: \n" +
               "yugabyted backup --database=northwind --cloud_storage_uri=s3://\n\n" +
               "# Backup a YCQL namespace into AWS S3 Bucket: \n" +
               "yugabyted backup --namespace=default --cloud_storage_uri=s3://\n\n",
    'restore' : "# Restore a YSQL Database into AWS S3 Bucket: \n" +
                "yugabyted restore --database=northwind --cloud_storage_uri=s3://\n\n" +
                "# Recover to a point in time for a database:\n" +
                "yugabyted restore " + \
                "--recover_to_point_in_time '2024-01-29 9:30:00 PM' --database yugabyte \n\n",
    'configure_read_replica' : "# Configure a new read replica cluster:\n" +
                  "yugabyted configure_read_replica new --rf=<replication_factor> " +
                  "--data_placement_constraint=<placement_policy_for_rr_cluster>\n\n" +
                  "# Modify an existing read replica cluster:\n" +
                  "yugabyted configure_read_replica modify --rf=<new_replication_factor> " +
                  "--data_placement_constraint=<new_placement_policy_for_rr_cluster>\n\n" +
                  "# Delete an existing read replica cluster:\n" +
                  "yugabyted configure_read_replica delete\n\n",
    "configure_read_replica_new" : "# Configure a new read replica cluster:\n" +
            "yugabyted configure_read_replica new --rf=<replication_factor> " +
            "--data_placement_constraint=<placement_policy_for_rr_cluster>",
    "configure_read_replica_modify" : "# Modify an existing read replica cluster:\n" +
               "yugabyted configure_read_replica modify --rf=<new_replication_factor> " +
               "--data_placement_constraint=<new_placement_policy_for_rr_cluster>",
    "configure_read_replica_delete" : "# Delete an existing read replica cluster:\n" +
               "yugabyted configure_read_replica delete",
    'configure' : "# Configure a multi-zone cluster:\n" +
                "yugabyted configure data_placement --fault_tolerance=zone\n\n" +
                "# Enable encryption at rest:\n" +
                "yugabyted configure encrypt_at_rest --enable\n\n" +
                "# Execute yb-admin command on the YugabyteDB cluster:\n" +
                "yugabyted configure admin_operation --command <yb-admin_command> \n\n" +
                "# Enable point-in-time recovery for a database:\n" +
                "yugabyted configure point_in_time_recovery --enable" + \
                " --retention <retention_period> --database <database_name> \n\n" +
                "For more examples use 'yugabyted configure [command] -h'\n\n",
    'configure_data_placement' : "# Configure a multi-zone cluster:\n" +
                       "yugabyted configure data_placement --fault_tolerance=zone\n\n" +
                       "# Configure a multi-region cluster:\n" +
                       "yugabyted configure data_placement --fault_tolerance=region\n\n" +
                       "# Configure a multi-zone cluster with the specified placement info and " + \
                       "rf:\n" + "yugabyted configure data_placement --fault_tolerance=zone " +
                       "--constraint_value=cloud1.region1.zone1,cloud2.region2.zone2," +
                       "cloud3.region3.zone3 --rf=3\n\n" +
                       "# Configure a multi-zone cluster with the order of preference" \
                       " to place the primary copy of the data:\n" +
                       "yugabyted configure data_placement  --constraint_value=" +
                       "cloud1.region1.zone1:<preference_number>," +
                       "cloud2.region2.zone2:<preference_number>," +
                       "cloud3.region3.zone3:<preference_number>\n\n",

    'configure_encrypt_at_rest' : "# Enable encryption at rest for a cluster:\n" +
                        "yugabyted configure encrypt_at_rest --enable\n\n" +
                        "# Disable encryption at rest for a cluster:\n" +
                        "yugabyted configure encrypt_at_rest --disable\n\n",
    'configure_admin_operation' : "# Execute yb-admin command on the YugabyteDB cluster:\n" +
                        "yugabyted configure admin_operation --command 'get_universe_config'\n\n",
    'configure_point_in_time_recovery' : "# Enable point-in-time recovery for a database:\n" +
                        "yugabyted configure point_in_time_recovery --enable" + \
                        " --retention <retention_period> --database <database_name> \n\n" +
                        "# Disable point-in-time recovery for a database:\n" +
                        "yugabyted configure point_in_time_recovery --disable" + \
                        " --database <database_name> \n\n" +
                        "# Display point-in-time recovery status for a YugabyteDB cluster:\n" +
                        "yugabyted configure point_in_time_recovery " + \
                        "--status \n\n",
    'upgrade' : "# Upgrade the YugabyteDB node:\n" +
                "yugabyted upgrade [ysql_catalog/finalize_new_version]\n\n",
    'upgrade_ysql_catalog' : "# yugabyted upgrade the ysql catalog: \n" +
                "yugabyted upgrade ysql_catalog [flags]\n\n",
    'upgrade_finalize_new_version' : "# Finalize the cluster upgrade process:\n" +
                         "yugabyted upgrade finalize_new_version\n\n" +
                        "# Finalize the cluster upgrade by specifying a timeout " +
                        "value for completion:\n" +
                         "yugabyted upgrade finalize_new_version --timeout <time_limit_ms>\n\n",
    'upgrade_check_version_compatibility' : "# Check if the cluster is compatible " +
                        "for upgrade to the new version\n" +
                        "yugabyted upgrade check_version_compatibility\n\n" +
                        "# Specifying a timeout value for completion:\n" +
                         "yugabyted upgrade check_version_compatibility " +
                         "--timeout <time_limit_ms>\n\n",
    'finalize_upgrade': "****** WARNING ******\n" +
                        "Deprecated. Please use 'upgrade finalize_new_version' command.\n\n",
    'xcluster' : "# Checkpoint a new xcluster replication:\n" +
                  "yugabyted xcluster create_checkpoint --replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_added_to_replication> " +
                  "[--automatic_mode]\n\n" +
                  "# Set-up xcluster replication:\n" +
                  "yugabyted xcluster set_up --replication_id=<replication_id> " +
                  "--target_address=<IP_of_any_target_node>\n\n" +
                  "# Show status of outbound and inbound replications:\n" +
                  "yugabyted xcluster status --replication_id=<replication_id>\n\n" +
                  "# Add a database to existing xcluster replication:\n" +
                  "## First add the database to the checkpoint:\n"
                  "yugabyted xcluster add_to_checkpoint --replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_added_to_replication>\n\n" +
                  "## Then add the database to the replication:\n" +
                  "yugabyted xcluster add_to_replication --replication_id=<replication_id> " +
                  "--databases=<database_to_be_added_to_replication> " +
                  "--target_address=<IP_of_any_target_node>\n\n"
                  "# Delete a xcluster replication:\n" +
                  "yugabyted xcluster status --replication_id=<replication_id> " +
                  "--target_address=<IP_of_any_target_node>\n\n" +
                  "# Delete databases from xcluster replication:\n" +
                  "yugabyted xcluster remove_database_from_replication " +
                  "--replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_removed_from_replication> " +
                  "--target_address=<IP_of_any_target_node>\n\n",
    "xcluster_create_checkpoint" : "# Checkpoint a new xcluster replication:\n" +
                  "yugabyted xcluster create_checkpoint --replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_added_to_replication> " +
                  "[--automatic_mode]\n\n",
    "xcluster_add_to_checkpoint" : "# Add databases to checkpoint:\n" +
                  "yugabyted xcluster add_to_checkpoint --replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_added_to_replication>\n\n",
    "xcluster_set_up" : "# Set-up xcluster replication:\n" +
                  "yugabyted xcluster set_up --replication_id=<replication_id> " +
                  "--target_address=<IP_of_any_target_node> --bootstrap_done\n\n",
    "xcluster_add_to_replication" : "# Add database to xcluster replication:\n" +
                  "yugabyted xcluster add_to_replication --replication_id=<replication_id> " +
                  "--database=<database_to_be_added_to_replication> " +
                  "--target_address=<IP_of_any_target_node> --bootstrap_done\n\n",
    "xcluster_status" : "# Show status of outbound and inbound replications:\n" +
                  "yugabyted xcluster status --replication_id=<replication_id>\n\n",
    "xcluster_delete_replication" : "# Delete a xcluster replication:\n" +
                  "yugabyted xcluster status --replication_id=<replication_id> " +
                  "--target_address=<IP_of_any_target_node>\n\n",
    "xcluster_remove_database_from_replication": "# Delete databases from xcluster replication:\n" +
                  "yugabyted xcluster remove_database_from_replication " +
                  "--replication_id=<replication_id> " +
                  "--databases=<comma_separated_databases_to_be_removed_from_replication> " +
                  "--target_address=<IP_of_any_target_node>\n\n",

}

EPILOG_COMMON = "Run '{} [command] -h' for help with specific commands.".format(SCRIPT_NAME)

EPILOG_SPECIFIC = {
    'start' : "Use conf file to configure advanced flags. Learn more about advanced flags " +
                "refer to the docs page: {}.\n\n".format(YUGABYTED_START),
    'stop' : "",
    'destroy' : "",
    'status' : "",
    'version' : "",
    'collect_logs' : "",
    'configure' : "",
    'cert' : "",
    "backup" : "",
    "restore" : "",
    "finalize_upgrade" : "",
}

# YugabyteDB configs.
IP_ANY = "0.0.0.0"
IP_LOCALHOST = "127.0.0.1"
DEFAULT_BIND_IP = IP_ANY
DEFAULT_MASTER_RPC_PORT = 7100
DEFAULT_TSERVER_RPC_PORT = 9100
DEFAULT_MASTER_WEBSERVER_PORT = 7000
DEFAULT_TSERVER_WEBSERVER_PORT = 9000
DEFAULT_YSQL_PORT = 5433
DEFAULT_YCQL_PORT = 9042
DEFAULT_YSQL_METRIC_PORT = 13000
DEFAULT_YCQL_METRIC_PORT = 12000
DEFAULT_WEBSERVER_PORT = 7200
DEFAULT_YUGABYTED_UI_PORT = 15433
DEFAULT_UPGRADE_YSQL_TIMEOUT = 60000
DEFAULT_CALLHOME = True
DEFAULT_YSQL_USER = "yugabyte"
DEFAULT_YSQL_PASSWORD = "yugabyte"
DEFAULT_YSQL_DB = "yugabyte"
DEFAULT_CLOUD_PROVIDER = "cloud1"
DEFAULT_CLOUD_REGION = "datacenter1"
DEFAULT_CLOUD_ZONE = "rack1"
YSQL_PASSWORD_LENGTH_WARNING = "Warning: Your 'YSQL_PASSWORD' length is greater than 99 characters.\
Please set 'PGPASSWORD' in environment variables to use 'bin/ysqlsh'."
DEFAULT_YCQL_USER = "cassandra"
DEFAULT_YCQL_PASSWORD = "cassandra"
DEFAULT_YCQL_KEYSPACE = None
VERSION_METADATA_PATH = os.path.join(YUGABYTE_DIR, "version_metadata.json")
YUGABYTE_API_CLIENT_PROGRAMS = {
    "ysql": "ysqlsh",
    "ycql": "ycqlsh",
}
YB_NUM_SHARDS_PER_TSERVER = 1
YSQL_NUM_SHARDS_PER_TSERVER = 1
METRICS_SNAPSHOT_LIST = [
    "handler_latency_yb_tserver_TabletServerService_Read_count",
    "handler_latency_yb_tserver_TabletServerService_Write_count",
    "handler_latency_yb_tserver_TabletServerService_Read_sum",
    "handler_latency_yb_tserver_TabletServerService_Write_sum",
    "disk_usage", "cpu_usage", "node_up"
]

# YugaWare configs. These have their own separate subdirectory to preserve our itest flow.
YUGAWARE_DIR = os.path.join(YUGABYTE_DIR, "ui")
YUGAWARE_BIN_DIR = os.path.join(YUGAWARE_DIR, "bin")
YUGAWARE_CONF = os.path.join(YUGAWARE_DIR, "conf/application.yugabyted.conf")
WEBSERVER_DB = "system_platform"
DEMO_DB_PREFIX = "yb_demo_"
YBC_NFS_DIR  = "/tmp/nfs"

BREW_CONF_FILE = "/usr/local/etc/yugabyted.conf"

ALERT_WARNING = "Warning"
ULIMIT_ERR_CODE = "LOW_ULIMITS"
TS_MASTER_ADDRS_FLAG = "tserver_master_addrs"

start_time_sec = time.time()

# Finds the path where a particular file is present from
# amongst the supplied paths.
def search_file_in_paths(dir_candidates, file_name):
    for candidate in dir_candidates:
        if os.path.exists(os.path.join(candidate, file_name)):
            Output.log("Found directory {} for"
                        " file {}".format(candidate, file_name))
            return os.path.join(candidate, file_name)

    # If post_install.sh script isn't found then don't error out
    # The caller assumes that the environment is dev and skips
    # performing the post installation steps
    if(file_name == "post_install.sh"):
        return None

    # If yugabyted-ui is not found, don't error out
    # Caller should set self.configs.saved_data.get("ui") to false if needed
    if(file_name == "yugabyted-ui"):
        Output.log(
            "Yugabyte {} file not found in paths {}. Please check "
            "the paths.".format(file_name, dir_candidates),
            logging.WARN
        )
        return None

    Output.log_error_and_exit(
        "Yugabyte {} file not found in paths {}. Please check "
        "the paths.".format(file_name, dir_candidates)
    )

# Checks if all given files exist in the given path
def check_files_in_path(path, files):
    has_error = False

    for file in files:
        if not os.path.exists(os.path.join(path, file)):
            has_error = True
            Output.log("{} file not found.".format(os.path.join(path, file)))

    return not has_error

def find_binary_path_in_test_env(binary_name):

    # Find directories inside YUGABYTE_DIR containing the specific binary.
    directories_with_binary = find_test_directories_with_binary(YUGABYTE_DIR, binary_name)
    if directories_with_binary:
        for directory in directories_with_binary:
            if 'postgres/bin' in directory:
                binary_path = os.path.join(directory, binary_name)
                Output.log("Using directory for running {}: {}".format(binary_name,
                                                                directory))
                return binary_path
    else:
        Output.log_error_and_exit(
            "No directories containing {} were found.".format(binary_name))

def find_test_directories_with_binary(root_dir, binary_name):
    directories_with_binary = []
    for dirpath, _, filenames in os.walk(root_dir):
        if binary_name in filenames:
            directories_with_binary.append(dirpath)
    return directories_with_binary

# Finds the path of a particular YB binary
def find_binary_location(binary_name):

    # Specific path for ysqlsh in test environment
    is_test_env = os.getenv('USER') == 'jenkins'
    if binary_name in ["ysqlsh", "pg_isready", "pg_upgrade"] and is_test_env:
        return find_binary_path_in_test_env(binary_name)

    # Default if tar is downloaded
    dir_candidates = [
        os.path.join(YUGABYTE_DIR, "bin")
    ]

    # Development environment
    dir_candidates += [
        os.path.join(YUGABYTE_DIR, "build", "latest", "bin"),
    ]

    # Development environment for UI
    dir_candidates += [
        os.path.join(YUGABYTE_DIR, "build", "latest", "gobin"),
    ]

    # Paths for pg_isready
    dir_candidates.extend([
        # If tar is downloaded
        os.path.join(YUGABYTE_DIR, "postgres", "bin"),
        # Development environment
        os.path.join(YUGABYTE_DIR, "build", "latest", "postgres", "bin")
    ])

    # Jenkins Test Environment
    dir_candidates += [
        os.path.join(YUGABYTE_JENKINS_BUILD_DIR, "bin")
    ]

    dir_candidates += [
        os.path.join(YUGABYTE_JENKINS_BUILD_DIR, "gobin")
    ]

    return search_file_in_paths(dir_candidates, binary_name)

# Extract or Download YBC package
def download_extract_ybc_binary():

    # Check for existence of YBC package
    ybc_file_pattern = "ybc-*.tar.gz"
    matching_files = glob.glob(ybc_file_pattern)
    result = True
    if len(matching_files) == 0:
        Output.log(
            "YBC binary archive file not found in {}.".
                format(YUGABYTE_DIR), logging.WARN
        )
        # check if Development environment
        if os.path.exists(os.path.join(YUGABYTE_DIR, "build", "latest", "bin")):
            Output.log(
                "Downloading YBC binary from {}.".
                    format(YBC_DOWNLOAD_LINK), logging.INFO)
            result = download_ybc_package()
            if result:
                matching_files = glob.glob(ybc_file_pattern)
            else:
                result = False
                return result
        else:
            # YBC binary not found in the YugabyteDB release package.
            result = False
            return result

    if len(matching_files) > 1:
        Output.log(
            "More than one YBC archive file found. Extracting the following {}.".
            format(matching_files[0], logging.WARN)
        )

    # Extract the YBC package into /ybc folder
    ybc_archive_path = YUGABYTE_DIR + "/" + matching_files[0]
    ybc_untar_path = YUGABYTE_DIR + "/ybc"
    with tarfile.open(ybc_archive_path, 'r:gz') as tar:
        for member in tar.getmembers():
            # Check if the member is a file (not a directory)
            if member.isfile():
                path_components = member.path.split(os.path.sep)
                # Remove the first folder path
                member.path = os.path.sep.join(path_components[1:])
                tar.extract(member, path=ybc_untar_path)

    return result

# Download YBC release from S3 bucket in dev environment
def download_ybc_package():
    try:
        response = urlopen(YBC_DOWNLOAD_LINK)
        CHUNK_SIZE = 1 << 20
        fileName = 'ybc-2.1.0.0-b9-linux-x86_64.tar.gz'
        Output.log('Downloading YBC Package: {}', YBC_DOWNLOAD_LINK)
        with open(fileName, 'wb') as f:
            while True:
                chunk = response.read(CHUNK_SIZE)
                if not chunk:
                    break
                f.write(chunk)
    except HTTPError as http_err:
        Output.log('HTTP error occurred while downloading YBC package: {}', http_err)
        return False
    except Exception as err:
        Output.log('Other error occurred while downloading YBC package current: {}', err)
        return False
    return True

# Find the yb controller binary
def find_ybc_binary_location(binary_name):
    # Default if tar is downloaded
    dir_candidates = [
        os.path.join(YUGABYTE_DIR, "ybc", "bin")
    ]

    return search_file_in_paths(dir_candidates, binary_name)

# Find the path of a particular postgres binary
def find_postgres_binary_location(binary_name):
    # Default if tar is downloaded
    dir_candidates = [
        os.path.join(YUGABYTE_DIR, "postgres", "bin"),
    ]

    # Development environment
    dir_candidates += [
        os.path.join(YUGABYTE_DIR, "build", "latest", "postgres", "bin"),
    ]
    return search_file_in_paths(dir_candidates, binary_name)


# Finds the path of the sample data
def find_sample_data_location(data_file):
    # Default if tar is downloaded
    dir_candidates = [
        os.path.join(YUGABYTE_DIR, "share")
    ]

    # Development environment
    dir_candidates += [
        os.path.join(YUGABYTE_DIR, "sample")
    ]

    return search_file_in_paths(dir_candidates, data_file)

# Finds the path of the version_metadata.json file
def find_version_metadata_location(version_file):
    # Default if tar is downloaded
    dir_candidates = [
        os.path.join(YUGABYTE_DIR)
    ]

    # Development environment
    dir_candidates += [
        os.path.join(YUGABYTE_DIR, "build", "latest")
    ]

    return search_file_in_paths(dir_candidates, version_file)

# Creates the Head Title for yugabyted CLI
def get_cli_title():
    title = Output.make_green(Output.make_green("Yugabyted CLI") + ": YugabyteDB command line")
    extra_len = len(Output.make_green(""))
    div_line = "+" + "-" * 98 + "+" + "\n"
    cli_title = div_line
    cli_title += ("| {:^" + str(105 + extra_len) + "} |\n").format(title)
    cli_title += div_line
    return cli_title

def using_time_sync_service():
    # List of recognized IP addresses and sources
    allow_list = ['169.254.169.123', 'metadata.google.internal', 'PHC',
                  'aws.com', 'google.com']

    cmd = ['chronyc', 'sources']
    out, _, ret_code = run_process(cmd, timeout=1, log_cmd=True)
    if ret_code == 0:
        for source in allow_list:
            if source in out:
                return True

    return False

def is_phc_configured():
    cmd = ['systemctl', 'status', 'clockbound']
    out, _, retcode = run_process(cmd, timeout=1, log_cmd=True)
    return retcode == 0 and 'PHC' in out

# Check if ip is ipv6
def is_ipv6(ip):
    try:
        ip_addr = ipaddress.ip_address(ip)
    except ValueError:
        return False
    if ip_addr.version == 6:
        return True
    return False

# Appends square brackets to ip if ipv6, otherwise does nothing
def get_url_from_ip(ip):
    if is_ipv6(ip):
        return "[" + str(ip) + "]"
    return str(ip)

class ControlScript(object):
    def __init__(self):
        self.configs = None
        self.processes = {}
        self.diagnostics = None
        self.alerts = []
        self.script = None
        self.setup_env_init = EnvBasedCredentials()
        self.callhome_threads = {}
        self.callhome_shutdown_events = {}

    # Starts YugabyteDB node.
    def start(self):
        if self.script.is_running():
            Output.print_out("{} is already running!".format(SCRIPT_NAME))
            sys.exit(1)

        Output.print_and_log("Starting {}...".format(SCRIPT_NAME))
        self.set_env_vars()

        if self.configs.temp_data.get("background"):
            # In daemon mode, self.daemonize() forks. The child process then executes
            # normal control flow. The parent process waits for the child process until
            # a status message can be printed to the terminal and then exits within daemonize.
            self.daemonize()
        self.script.write_pid(os.getpid())

        self.set_signals(self.kill_children)
        atexit.register(self.kill_children)
        Output.script_exit_func = self.kill_children

        if self.configs.saved_data.get("read_replica"):
            self.start_rr_process()
        else:
            self.start_processes()

    # Kills all processes related to yugabyted
    def kill_all_procs(self):
        # Shutdown all callhome threads gracefully
        self.shutdown_callhome_thread(timeout=5)

        pid = self.script.get_pid()
        if pid:
            pgid = os.getpgid(pid)
            if not pgid:
                Output.log("PGID could not be found for {} process".format(SCRIPT_NAME) +
                    "with PID {}. Is {} running?".format(pid, SCRIPT_NAME))
                return ("No PGID", pid)
            else:
                try:
                    os.killpg(pgid, SIGTERM)
                except OSError as err:
                    return (err, pid)
        self.script.delete_pidfile()
        return (None, pid)

    # Kills currently running yugabyted process if it exists.
    def stop(self, *args):

        if len(os.listdir(self.configs.saved_data.get("data_dir"))) == 0:
            Output.print_out("{} is not running.".format(SCRIPT_NAME))
            sys.exit(1)

        upgrade_in_progress = self.configs.saved_data.get("upgrade_in_progress")
        catalog_upgrade_completed = self.configs.saved_data.get("catalog_upgrade_completed")

        if upgrade_in_progress and not catalog_upgrade_completed:
            tserver_pidfile =  os.path.join(self.configs.saved_data.get("data_dir"),
                                            "{}.pid".format("tserver"))
            tserver_pid_value = self.read_pidfile(tserver_pidfile)
            old_release_tserver_cmdline = self.get_cmdline_from_pid(tserver_pid_value)
            self.configs.saved_data["old_release_cmd"] = old_release_tserver_cmdline
            self.configs.save_configs()
            master_addresses = self.configs.saved_data.get("current_masters")
            if master_addresses == '':
                Output.log_error_and_exit(Output.make_red("Error") +
                                          ": cannot retreive the masters.")

        (err, pid) = self.kill_all_procs()
        if err:
            Output.log_error_and_exit(
                "Failed to shut down {}: {}. Please check PID in {}".format(
                    SCRIPT_NAME, err, self.script.pidfile))
        elif pid:
            self.script.wait_until_stop(pid)
            Output.print_out("Stopped {} using config {}.".format(SCRIPT_NAME, self.conf_file))
        sys.exit(0)

    # Back up a database.
    def backup(self):

        status_details = []
        status_display_info = {}

        if not self.configs.saved_data.get("backup_daemon"):
            final_status = "Backup daemon process not found. " + \
                            "Please restart the node with --backup_daemon=true."
            status_details = [
                (Output.make_red("Error"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return
        else:
            YBControllerCLIProxy.init()
            YBControllerCLIProxy.set_certs_dir(self.configs.saved_data.get("master_flags"), \
                self.configs.saved_data.get("secure"), self.configs.saved_data.get("certs_dir"))

        ybc_server_dir = "{}/yb-data/ybc".format(self.configs.saved_data.get("data_dir"))
        ybc_task_type = "backup"

        backup_args = dict()
        backup_args["cloud_type"] = self.configs.temp_data.get("ybc_cloud_type")
        backup_args["bucket"] = self.configs.temp_data.get("ybc_cloud_storage_bucket")
        backup_args["cloud_dir"] = self.configs.temp_data.get("ybc_cloud_storage_dir")
        backup_args["ybc_task"] = ybc_task_type
        if self.configs.temp_data.get("database_name_backup_restore"):
            backup_args["ns"] = self.configs.temp_data.get("database_name_backup_restore")
            backup_args["ns_type"] = "ysql"
        if self.configs.temp_data.get("keyspace_name_backup_restore"):
            backup_args["ns"] = self.configs.temp_data.get("keyspace_name_backup_restore")
            backup_args["ns_type"] = "ycql"

        if OS_NAME != "Linux":
            final_status = "Backup command is not supported in {} enviroments.".format(OS_NAME)
            status_details = [
                (Output.make_yellow("Status"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        if self.configs.temp_data.get("ybc_status"):
            final_status = ""
            backup_status = YBControllerCLIProxy.check_for_ongoing_ybc_task(
                backup_args, ybc_server_dir)
            if backup_status is not None:
                if backup_status == "OK":
                    final_status = "Backup is complete."
                elif backup_status == "COMMAND_FAILED":
                    final_status = "Backup has failed. " + \
                        "Check the logs for more details."
                    status_display_info[final_status] = Output.make_red
                elif len(backup_status) == 0:
                    final_status = "Backup is in Progress. Check the logs for more details."
                else:
                    final_status = "Backup status: {}".format(backup_status)
            else:
                if YBControllerCLIProxy.check_for_existing_backup(backup_args):
                    final_status = "Backup is complete."
                else:
                    final_status = "Backup not found in the given cloud location."
                    status_display_info[final_status] = Output.make_red

            status_details = [
                    (Output.make_yellow("Status"), final_status)
                ]
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        # Check if there is a currently running backup task on the local node.
        backup_status = YBControllerCLIProxy.check_for_ongoing_ybc_task(
            backup_args, ybc_server_dir)
        if backup_status is not None:
            if backup_status in ["OK", "NOT_FOUND", "COMMAND_FAILED"]:
                YBControllerCLIProxy.rename_ybc_lock_file(backup_args, ybc_server_dir)
            else:
                final_status = "Backup is in progress."
                status_details = [
                    (Output.make_yellow("Status"), final_status)
                ]
                Output.print_out(self.get_status_string_common(status_details))
                return

        # Todo: check if there is any ongoing backup tasks on other nodes.

        # Check if the cloud location has any other previous backup.
        if YBControllerCLIProxy.check_for_existing_backup(backup_args):
            final_status = "Already a backup is present " +  \
                            "in the cloud location {}. Abort!!".format( backup_args["cloud_dir"])
            status_details = [
                (Output.make_yellow("Status"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return


        advertise_ip = self.advertise_ip()
        backup_task_id = YBControllerCLIProxy.backup_to_cloud(
            advertise_ip, backup_args
        )
        if not backup_task_id:
            final_status = "Backup failed. Check the logs."
            status_details = [
                (Output.make_yellow("Status"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        YBControllerCLIProxy.write_ybc_lock_file(
            backup_args, backup_task_id, ybc_server_dir, advertise_ip
        )
        final_status = "Backup started successfully."
        Output.log("Backup started successfully with Task UUID: {}".format(backup_task_id))

        # Send callhome event for backup
        self.start_callhome_thread(
            CALLHOME_EVENT_BACKUP,
            lambda: self._send_backup_restore_callhome(CALLHOME_EVENT_BACKUP,
                                                         backup_task_id))

        status_details = [
            (Output.make_yellow("Status"), final_status)
        ]
        Output.print_out(self.get_status_string_common(status_details))
        self.shutdown_callhome_thread(CALLHOME_EVENT_BACKUP, timeout=5)

    # PITR restore
    def restore_pitr(self):
        if self.configs.temp_data.get("database_name_backup_restore"):
            pitr_object = "ysql.{}".format(self.configs.temp_data.get(
                                                "database_name_backup_restore"))
            pitr_object_name = pitr_object.split('.', 1)[1]
            pitr_yb_api_type = "database"
        elif self.configs.temp_data.get("keyspace_name_backup_restore"):
            pitr_object = "ycql.{}".format(self.configs.temp_data.get(
                                                "keyspace_name_backup_restore"))
            pitr_object_name = pitr_object.split('.', 1)[1]
            pitr_yb_api_type = "keyspace"

        Output.init_animation("Verifying Point-In-Time Recovery configs...")
        master_addresses = self.configs.saved_data.get("current_masters")
        schedules = YBAdminProxy.list_schedules(master_addresses)
        if schedules is None:
            Output.log_error_and_exit(Output.update_animation("Failed to verify"
            " Point-In-Time Recovery configs.", status=Output.ANIMATION_FAIL))

        pitr_object_exists, is_pitr_enabled = self.validate_pitr_configs(pitr_object, schedules)
        if not pitr_object_exists:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": The specified {} {} does not exist. Please "
            "specify a valid {} to recover to a point in time.".format(
                                        pitr_yb_api_type, pitr_object_name, pitr_yb_api_type)
            )
        if not is_pitr_enabled:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": Point-in-time recovery is not enabled"
            " for {} {}. Please enable point-in-time recovery"
            " before recovering to a point in time.".format(pitr_yb_api_type, pitr_object_name)
            )
        restore_time = self.configs.temp_data.get("restore_time")
        # Get the schedule ID to perform pitr
        pitr_object_id, earliest_recoverable_time, _ = self.get_pitr_object_details(
                                                        pitr_object, schedules)
        valid, error_message = self.validate_restore_time(earliest_recoverable_time, restore_time)
        if not valid:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                    status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(Output.make_red("ERROR") + ": " + error_message)

        Output.update_animation("Verified Point-In-Time Recovery configs.")
        Output.init_animation("Recovering to a Point-In-Time...")
        restore_time_object = datetime.strptime(restore_time, "%Y-%m-%d %I:%M:%S %p")
        ret_code, err = YBAdminProxy.restore_snapshot_schedule(master_addresses,
                pitr_object_id, restore_time_object.strftime("%Y-%m-%d %H:%M:%S"))

        # Catch specific errors
        if ret_code:
            if "Cannot perform a forward restore" in err:
                # Extract the specific times using regex
                times = re.findall(r'\{ days: (\d+) time: ([^}]+) \}', err)
                # Convert and format the times
                collected_times = []
                for days, time in times:
                    days = int(days)
                    date = datetime(1970, 1, 1) + timedelta(days=days)
                    # Check if the time string includes fractional seconds
                    has_fractional = '.' in time
                    if has_fractional:
                        time_format = "%Y-%m-%d %H:%M:%S.%f"
                    else:
                        time_format = "%Y-%m-%d %H:%M:%S"

                    date_time = datetime.strptime('{} {}'.format(date.date(), time), time_format)
                    # If there is a fractional part, zero out the microseconds
                    if has_fractional:
                        date_time = date_time.replace(microsecond=0)
                    collected_times.append(date_time.strftime("%Y-%m-%d %I:%M:%S %p"))

                Output.update_animation("Recovery Failed.",
                        status=Output.ANIMATION_FAIL)
                Output.log_error_and_exit(Output.make_red("FAILED") +
                    ": The requested restoration time {} "
                    "is in between a previous restored to and restored from time. "
                    "Please do not specify a time between {} and {}."
                    .format(collected_times[2], collected_times[0], collected_times[1])
                )
            elif "YSQL upgrade" in err:
                Output.update_animation("Recovery Failed.",
                        status=Output.ANIMATION_FAIL)
                Output.log_error_and_exit(Output.make_red("FAILED") +
                    ": The requested restoration time {} is before performing a"
                    " YSQl upgrade. Please specify a restoration time after the upgrade"
                    " was completed.".format(restore_time)
                )
            else:
                Output.log(err)
                Output.log_error_and_exit(Output.update_animation("Recovery Failed.",
                        status=Output.ANIMATION_FAIL))
        else:
            Output.log("Successfully recovered to a point in time.")

        Output.update_animation("Successfully recovered to the specified Point-In-Time.")
        status_details = self.get_pitr_config_details(pitr_object, restore_time = restore_time)

        Output.print_out(self.get_status_string_common(status_details[0], status_details[1]))
        sys.exit(0)

    # Restore a database.
    def restore(self):

        status_details = []
        status_display_info = {}

        if self.configs.temp_data.get("restore_time"):
            self.restore_pitr()

        if not self.configs.saved_data.get("backup_daemon"):
            final_status = "Backup daemon process not found. " + \
                            "Please restart the node with --backup_daemon=true."
            status_details = [
                (Output.make_red("Error"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return
        else:
            YBControllerCLIProxy.init()
            YBControllerCLIProxy.set_certs_dir(self.configs.saved_data.get("master_flags"), \
                self.configs.saved_data.get("secure"), self.configs.saved_data.get("certs_dir"))

        ybc_server_dir = "{}/yb-data/ybc".format(self.configs.saved_data.get("data_dir"))
        ybc_task_type = "restore"

        restore_args = dict()
        restore_args["cloud_type"] = self.configs.temp_data.get("ybc_cloud_type")
        restore_args["bucket"] = self.configs.temp_data.get("ybc_cloud_storage_bucket")
        restore_args["cloud_dir"] = self.configs.temp_data.get("ybc_cloud_storage_dir")
        restore_args["ybc_task"] = ybc_task_type

        if self.configs.temp_data.get("database_name_backup_restore"):
            restore_args["ns"] = self.configs.temp_data.get("database_name_backup_restore")
            restore_args["ns_type"] = "ysql"
        if self.configs.temp_data.get("keyspace_name_backup_restore"):
            restore_args["ns"] = self.configs.temp_data.get("keyspace_name_backup_restore")
            restore_args["ns_type"] = "ycql"

        if OS_NAME != "Linux":
            final_status = "Restore command is not supported in {} enviroments.".format(OS_NAME)
            status_details = [
                (Output.make_yellow("Status"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        if self.configs.temp_data.get("ybc_status"):
            final_status = ""
            restore_status = YBControllerCLIProxy.check_for_ongoing_ybc_task(
                restore_args, ybc_server_dir)
            if restore_status is not None:
                if restore_status == "OK":
                    final_status = "Restore is complete."
                elif restore_status == "COMMAND_FAILED":
                    final_status = "Restore has failed. Check the logs for more details."
                    status_display_info[final_status] = Output.make_red
                elif not restore_status:
                    final_status = "Restore is in progress."
                else:
                    final_status = "Restore status: {}".format(restore_status)
            else:
                final_status = "No restore task found for {}".format(restore_args["ns"])
                status_display_info[final_status] = Output.make_red

            status_details = [
                    (Output.make_yellow("Status"), final_status)
                ]
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        # Check if the cloud location has a backup.
        if not YBControllerCLIProxy.check_for_existing_backup(restore_args):
            final_status = "Backup not present in the given " + \
                "cloud location {}. Abort!!".format(restore_args["cloud_dir"])
            status_details = [
                    (Output.make_yellow("Status"), final_status)
                ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            return

        # Check if there is a currently running backup task on the local node.
        ybc_task_status = YBControllerCLIProxy.check_for_ongoing_ybc_task(
            restore_args, ybc_server_dir)
        if ybc_task_status is not None:
            if ybc_task_status in ["OK"]:
                YBControllerCLIProxy.rename_ybc_lock_file(restore_args, ybc_server_dir)
            else:
                final_status = "backup/restore operation is in progress. " + \
                        "Check the logs for more details."
                status_details = [
                    (Output.make_yellow("Status"), final_status)
                ]
                status_display_info[final_status] = Output.make_red
                Output.print_out(self.get_status_string_common(status_details,
                                                               status_display_info))
                return


        advertise_ip = self.advertise_ip()
        restore_task_id = YBControllerCLIProxy.restore_from_cloud(
            advertise_ip, restore_args
        )
        if not restore_task_id:
            final_status = "Restore has failed. Check the logs for more details."
            status_details = [
                (Output.make_yellow("Status"), final_status)
            ]
            status_display_info[final_status] = Output.make_red
            Output.print_out(self.get_status_string_common(status_details,
                                                            status_display_info))
            return

        YBControllerCLIProxy.write_ybc_lock_file(
            restore_args, restore_task_id, ybc_server_dir, advertise_ip
        )
        final_status = "Restore started successfully."
        Output.log("Restore started successfully with Task UUID: {}".format(restore_task_id))

        # Send callhome event for restore
        self.start_callhome_thread(
            CALLHOME_EVENT_RESTORE,
            lambda: self._send_backup_restore_callhome(CALLHOME_EVENT_RESTORE,
                                                        restore_task_id))

        status_details = [
            (Output.make_yellow("Status"), final_status)
        ]
        Output.print_out(self.get_status_string_common(status_details))
        self.shutdown_callhome_thread(CALLHOME_EVENT_RESTORE, timeout=5)


    # Prints status of YugabyteDB.
    def status(self):
        if len(os.listdir(self.configs.saved_data.get("data_dir"))) != 0:
            Output.init_animation("Fetching status...")
            status_output, ret_code = self.get_status_string()
            Output.update_animation("", Output.ANIMATION_STOP)
            Output.print_out("\n" + status_output.strip())
            if ret_code:
                sys.exit(ret_code)
        else:
            Output.print_out("{} is not running.".format(SCRIPT_NAME))

    # Destroy the YugabyteDB cluster.
    def destroy(self):
        (err, pid) = self.kill_all_procs()
        if err:
            Output.log_error_and_exit(
                "Failed to shut down {}: {}. Please check PID in {}".format(
                    SCRIPT_NAME, err, self.script.pidfile))
        elif pid:
            self.script.wait_until_stop(pid)
            Output.print_out("Stopped {} using config {}.".format(SCRIPT_NAME, self.conf_file))
        logpath = self.configs.saved_data.get("log_dir")
        datapath = self.configs.saved_data.get("data_dir")
        additional_datapaths = self.configs.saved_data.get(
                                            "additional_data_dir").split(',')
        gen_certs_dir = self.configs.saved_data.get("gen_certs_dir")
        certs_dir = self.configs.saved_data.get("certs_dir")
        config_path = os.path.dirname(self.conf_file)

        if (self.conf_file == BREW_CONF_FILE):
            Output.print_out("{} destroy is not supported for brew installations.".format(
                SCRIPT_NAME))
            return

        if os.path.isdir(logpath):
            shutil.rmtree(logpath)
            Output.print_out("Deleted logs at {}.".format(logpath))

        if os.path.isdir(datapath):
            shutil.rmtree(datapath)
            Output.print_out("Deleted data at {}.".format(datapath))

        is_additional_datapath_deleted = False
        for path in additional_datapaths:
            if os.path.isdir(path):
                is_additional_datapath_deleted = True
                shutil.rmtree(path)

        if is_additional_datapath_deleted:
            Output.print_out("Deleted additional data at {}".format(
                                                ", ".join(additional_datapaths)))

        if os.path.isdir(gen_certs_dir):
            shutil.rmtree(gen_certs_dir)
            Output.print_out("Deleted generated certs at {}.".format(gen_certs_dir))

        if os.path.isdir(certs_dir):
            shutil.rmtree(certs_dir)
            Output.print_out("Deleted certs at {}.".format(certs_dir))

        if os.path.isdir(config_path):
            shutil.rmtree(config_path)
            Output.print_out("Deleted conf at {}.".format(config_path))

        sys.exit(0)

    # Prints YugabyteDB version.
    def version(self):
        VERSION_METADATA_PATH = find_version_metadata_location("version_metadata.json")
        with open(VERSION_METADATA_PATH) as metadata:
            data = json.load(metadata)
            title = "Version".format(SCRIPT_NAME)
            output = "\n" + "-" * 70 + "\n"
            output += ("| {:^66} |\n").format(title)
            output += "-" * 70 + "\n"
            build = data.get("build_number")
            try:
                version = "{}-b{}".format(data.get("version_number"), int(build))
            except ValueError as e:
                version = "{} ({})".format(data.get("version_number"), build)
            for k, v in [
                    ("Version", version),
                    ("Build Time", data.get("build_timestamp")),
                    ("Build Hash", data.get("git_hash"))]:
                output_k = Output.make_yellow(k)
                extra_len = len(Output.make_yellow(""))
                output += ("| {:" + str(15 + extra_len) + "}: {:<49} |\n").format(output_k, v)
            output += "-" * 70 + "\n"
            Output.print_out(output)

    def upgrade_check_version_compatibility(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        if self.configs.saved_data.get("secure") and not os.path.exists(
            os.path.expanduser("~/.pgpass")):
                Output.log_error_and_exit(Output.make_red("ERROR") +
                            ": .pgpass file is not present in the home directory.")

        advertise_ip = self.advertise_ip()
        ysql_port = self.configs.saved_data.get("ysql_port")
        data_dir = self.configs.saved_data.get("data_dir")
        Output.init_animation("Checking version compatibility...")

        timeout = 60000
        if self.configs.temp_data.get("upgrade_timeout") is not None:
            timeout = self.configs.temp_data.get("upgrade_timeout")

        start_time = time.time()
        upgrade_check_message, upgrade_check_status = (
            PGUpgradeProxy.check_for_pg_upgrade_compatibility(advertise_ip,
                                ysql_port, data_dir, timeout))
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)

        compatible = (upgrade_check_status == 0)
        # Send callhome event for upgrade check version compatibility
        self.start_callhome_thread(
            CALLHOME_EVENT_UPGRADE_CHECK_VERSION_COMPATIBILITY,
            lambda: self._send_upgrade_check_version_compatibility_callhome(
                timeout, compatible, time_taken))
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_UPGRADE_CHECK_VERSION_COMPATIBILITY, timeout=5)

        if upgrade_check_status:
            Output.log_error_and_exit("FAILED: PG upgrade compatibility check failed. " +
                                    "Please check the logs. \n{}".format(upgrade_check_message))

        Output.update_animation("Clusters are compatible for upgrade.")

    def upgrade_finalize_new_version(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        tserver_addresses = self.get_tserver_addresses()

        # Verify if all nodes were upgraded successfully
        Output.init_animation("Verifying version compatibility across all nodes...")

        node_versions = self.get_all_nodes_version(tserver_addresses)
        highest_version = self.get_highest_version(node_versions)
        mismatched_nodes_version = self.verify_all_nodes_version(node_versions,
                                                            highest_version)

        if len(mismatched_nodes_version) > 0:
            mismatched_nodes = ', '.join(mismatched_nodes_version)
            Output.update_animation("Version compatibility verification failed.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("Error") + ": Version mismatch detected in the " +
            "following nodes: {}. Please upgrade these".format(mismatched_nodes) +
            " nodes to version {} and then rerun the".format(highest_version) +
            Output.make_yellow(" yugabyted finalize_upgrade") + " command."
        )

        # All nodes match the correct version
        # Run post-upgrade tasks
        Output.update_animation("Verified version compatibility across all nodes.")
        master_addresses = self.configs.saved_data.get("current_masters")

        timeout = 360000
        if self.configs.temp_data.get("upgrade_timeout") is not None:
            timeout = self.configs.temp_data.get("upgrade_timeout")

        Output.init_animation("Finalizing the upgrade to {}...".format(highest_version))
        start_time = time.time()
        success = YBAdminProxy.finalize_upgrade_uber_command(master_addresses, timeout)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        node_count = len(tserver_addresses) if tserver_addresses else 0
        failure_reason = None

        if not success:
            failure_reason = "Failed to finalize upgrade"
            # Send callhome event before exit
            self.start_callhome_thread(
                CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION,
                lambda: self._send_upgrade_finalize_new_version_callhome(
                    highest_version, node_count, timeout, success,
                    failure_reason, time_taken))
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION, timeout=5)
            Output.log_error_and_exit(Output.update_animation("Failed to "
                "Finalize the upgrade.",status=Output.ANIMATION_FAIL))
        else:
            Output.log("Upgrade finalize step was successful.")

        Output.update_animation("Cluster Upgrade Successfully " +
                                    "Completed to {}.".format(highest_version))
        # Send callhome event for upgrade finalize new version
        self.start_callhome_thread(
            CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION,
            lambda: self._send_upgrade_finalize_new_version_callhome(
                highest_version, node_count, timeout, success,
                failure_reason, time_taken))
        status_details = []
        status_display_info = {}
        final_status = "Post-upgrade tasks completed successfully."
        status_details = [
                    (Output.make_yellow("Status"), final_status)]
        status_display_info[final_status] = Output.make_green
        status_details.append((Output.make_yellow("Version"), highest_version))
        Output.print_out(self.get_status_string_common(status_details, status_display_info))
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION, timeout=5)

    # Finalize YugabyteDB cluster upgrade
    # to-do: Need to deprecated this method
    def finalize_upgrade(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        tserver_addresses = self.get_tserver_addresses()

        # Verify if all nodes were upgraded successfully
        Output.init_animation("Verifying version compatibility across all nodes...")
        node_versions = self.get_all_nodes_version(tserver_addresses)
        highest_version = self.get_highest_version(node_versions)
        mismatched_nodes_version = self.verify_all_nodes_version(node_versions,
                                                            highest_version)

        if len(mismatched_nodes_version) > 0:
            mismatched_nodes = ', '.join(mismatched_nodes_version)
            Output.update_animation("Version compatibility verification failed.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("Error") + ": Version mismatch detected in the " +
            "following nodes: {}. Please upgrade these".format(mismatched_nodes) +
            " nodes to version {} and then rerun the".format(highest_version) +
            Output.make_yellow(" yugabyted finalize_upgrade") + " command."
        )

        # All nodes match the correct version
        # Run post-upgrade tasks
        Output.update_animation("Verified version compatibility across all nodes.")
        master_addresses = self.configs.saved_data.get("current_masters")
        upgrade_ysql_timeout = self.configs.temp_data.get("upgrade_ysql_timeout")

        Output.init_animation("Promoting Auto Flags...")
        if not YBAdminProxy.promote_auto_flags(master_addresses):
            Output.log_error_and_exit(Output.update_animation("Failed to "
                "Promote Auto Flags.",status=Output.ANIMATION_FAIL))
        else:
            Output.log("Promote auto flags step was successful.")

        Output.update_animation("Successfully Promoted Auto Flags.")
        Output.init_animation("Upgrading YSQL...")
        if not YBAdminProxy.upgrade_ysql(master_addresses,
                                    upgrade_ysql_timeout):
            Output.log_error_and_exit(Output.update_animation("Failed to "
                "Upgrade YSQL.",status=Output.ANIMATION_FAIL))
        else:
            Output.log("Upgrade YSQL step was successful.")

        Output.update_animation("Successfully Upgraded YSQL.")
        status_details = []
        status_display_info = {}
        final_status = "Post-upgrade tasks completed successfully."
        status_details = [
                    (Output.make_yellow("Status"), final_status)]
        status_display_info[final_status] = Output.make_green
        status_details.append((Output.make_yellow("Version"), highest_version))
        Output.print_out(self.get_status_string_common(status_details, status_display_info))

    # Starts an interactive YSQL shell.
    def connect_ysql(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{} is not running. Cannot connect to YSQL.".format(SCRIPT_NAME))
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        is_password_passed=False
        for arg in sys.argv[1:]:
            if arg.startswith("--password"):
                is_password_passed = True
                break

        if self.configs.saved_data.get("secure") and is_password_passed:
            ysql_proxy.connect_with_password()
        else:
            ysql_proxy.connect_without_password()

    # Starts an interactive YCQL shell.
    def connect_ycql(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{} is not running. Cannot connect to YCQL.".format(SCRIPT_NAME))
        if self.configs.saved_data.get("secure"):
            self.setup_env_init.setup_cert_file_path(self.configs.
                            saved_data.get("ca_cert_file_path"))
        ycql_proxy = YcqlProxy(ip=self.advertise_ip(),
                        port=self.configs.saved_data.get("ycql_port"),
                        secure=self.configs.saved_data.get("secure"))
        is_password_passed=False
        for arg in sys.argv[1:]:
            if arg.startswith("--password"):
                is_password_passed = True
                break

        if self.configs.saved_data.get("secure") and is_password_passed:
            ycql_proxy.connect_with_password()
        else:
            ycql_proxy.connect_without_password()

    # Creates demo database and starts an interactive shell into it. Destroys the sample database
    # after shell quits.
    def demo(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{0} is not running. Please run `{0} start` before starting a demo.".format(
                    SCRIPT_NAME))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use demo commands " +
                                      "from a read replica node.")

        db_name = DEMO_DB_PREFIX + self.configs.temp_data.get("demo_db")
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        if ysql_proxy.db_exists(db_name):
            Output.log_error_and_exit(
                "Demo is already running. Concurrent demos are currently unsupported.")
        # TODO: Race condition currently exists when running demo too close to each other. This
        # will be solved when concurrent isolated demos are implemented.
        Output.print_out("Now creating demo database")
        self.create_demo()

        # Ignore kill SIGINT to match normal ysqlsh and psql behavior.
        signal(SIGINT, SIG_IGN)
        signal(SIGABRT, self.destroy_demo)
        signal(SIGTERM, self.destroy_demo)
        atexit.register(self.destroy_demo)
        self.connect_demo()
        self.set_signals(SIG_DFL)

    # Create target demo database if it does not exist.
    def create_demo(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{0} is not running. Please run `{0} start` before starting a demo.".format(
                    SCRIPT_NAME))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use demo commands " +
                                      "from a read replica node.")

        demo_db = self.configs.temp_data.get("demo_db")
        db_name = DEMO_DB_PREFIX + demo_db
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        if ysql_proxy.db_exists(db_name):
            Output.print_out("Demo database {} already exists.".format(demo_db))
            return

        Output.print_out(
            "Initializing {} demo database. This may take up to a minute...".format(demo_db))
        # Create demo database.
        Output.log("Creating database {}...".format(db_name))
        ysql_proxy.create_db(db_name)

        # Populate demo database.
        Output.log("Populating {} with sample data...".format(db_name))
        files = []
        for name in Configs.get_demo_info()[demo_db]["files"]:
            files.append(os.path.join(find_sample_data_location(name)))
        ysql_proxy.load_files(files, db=db_name)

        msg = "Successfully loaded sample database!"
        Output.print_and_log(msg)

    # Run YSQL shell in target demo database.
    def connect_demo(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{0} is not running. Please run `{0} start` before starting a demo.".format(
                    SCRIPT_NAME))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use demo commands " +
                                      "from a read replica node.")

        demo_db = self.configs.temp_data.get("demo_db")
        db_name = DEMO_DB_PREFIX + demo_db
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        start_time = time.time()
        if not ysql_proxy.db_exists(db_name):
            self.create_demo()
            end_time = time.time()
        else:
            end_time = start_time
        time_taken = self._format_time_taken_ms(start_time, end_time)

        # Ignore kill SIGINT to match normal ysqlsh and psql behavior.
        signal(SIGINT, SIG_IGN)
        # Send callhome event for demo connect
        self.start_callhome_thread(
            CALLHOME_EVENT_DEMO_CONNECT,
            lambda: self._send_demo_connect_callhome(demo_db, time_taken))
        website = Output.make_underline(SAMPLE_DATA_LINKS[demo_db])
        Output.print_out(Configs.get_demo_info()[demo_db]["examples"])
        Output.print_out(
            "For more, go to {}\n".format(website)
        )
        ysql_proxy.connect_with_password(db=db_name)
        self.shutdown_callhome_thread(CALLHOME_EVENT_DEMO_CONNECT, timeout=5)

    # Destroy target demo database if it exists.
    def destroy_demo(self, signum=None, frame=None):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{0} is not running. Please run `{0} start` before starting a demo.".format(
                    SCRIPT_NAME))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use demo commands " +
                                      "from a read replica node.")

        demo_db = self.configs.temp_data.get("demo_db")
        db_name = DEMO_DB_PREFIX + demo_db
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        if ysql_proxy.db_exists(db_name):
            ysql_proxy.drop_db(db_name)
        msg = "Deleted demo database {}.".format(demo_db)
        Output.print_and_log(msg)

    def collect_logs(self):
        logpath = self.configs.saved_data.get("log_dir")
        collect_at_dir=self.configs.temp_data.get("collect_at_dir")
        if not os.path.exists(logpath):
            Output.print_and_log("No logs directory at {}".format(logpath))
            return
        tmpprefix = "yugabyted-{}s.tar.gz".format(datetime.now().strftime("%Y-%m-%d-%Hh%Mm%S.%f"))
        tarpath = os.path.join(collect_at_dir, tmpprefix)
        with tarfile.open(name=tarpath, mode='w:gz', dereference=True) as archive:
            for item in os.listdir(logpath):
                item_path = os.path.join(logpath, item)
                arcname = os.path.relpath(item_path, logpath)
                archive.add(item_path, arcname=arcname)
        status_details = [
                (Output.make_yellow("Status"),"Logs collected successfully.")
        ]
        if self.configs.temp_data.get("collect_logs_stdout"):
            status_details = [
                (Output.make_yellow("Status"), "Logs collected successfully." + \
                    "Find the displayed logs below.")
            ]
            Output.log("Logs are packaged into {}".format(tarpath))
            if tarfile.is_tarfile(tarpath):
                with open(tarpath, 'rb') as tar_fd:
                    if sys.version_info[0] == 3:
                        sys.stdout.buffer.write(tar_fd.read())
                    else:
                        sys.stdout.write(tar_fd.read())
        display_msg= "Collected logs can be found at: {}".format(tarpath)
        status_details.extend([
                (Output.make_yellow("Collected logs path"), display_msg)
        ])
        Output.print_and_log(self.get_status_string_common(status_details))

    # Configuring PITR in YugabyteDB cluster
    def configure_point_in_time_recovery(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        if self.configs.temp_data.get("database_name_backup_restore"):
            pitr_object = "ysql.{}".format(self.configs.temp_data.get(
                                                "database_name_backup_restore"))
            pitr_yb_api_type = "database"
        elif self.configs.temp_data.get("keyspace_name_backup_restore"):
            pitr_object = "ycql.{}".format(self.configs.temp_data.get(
                                                "keyspace_name_backup_restore"))
            pitr_yb_api_type = "keyspace"
        else:
            pitr_object = None
            pitr_yb_api_type = None

        Output.init_animation("Verifying Point-In-Time Recovery configs...")
        master_addresses = self.configs.saved_data.get("current_masters")
        schedules = YBAdminProxy.list_schedules(master_addresses)
        if schedules is None:
            Output.log("Failed to retrieve Point-In-Time Recovery schedules")
            Output.log_error_and_exit(Output.update_animation("Failed to verify"
            " Point-In-Time Recovery configs.", status=Output.ANIMATION_FAIL))

        # Enable PITR
        if self.configs.temp_data.get("enable_pitr"):
            self.enable_pitr(pitr_object, pitr_yb_api_type, schedules)
        # Disable PITR
        elif self.configs.temp_data.get("disable_pitr"):
            self.disable_pitr(pitr_object, pitr_yb_api_type, schedules)
        # Show PITR status
        else:
            self.display_pitr(pitr_object, pitr_yb_api_type, schedules)

    def get_complete_bootstrap_msg(self, dbs_to_be_bootstrapped, dbs_not_to_be_bootstrapped):
        complete_bootstrap_err_msg = ""
        if dbs_to_be_bootstrapped and len(dbs_to_be_bootstrapped) != 0:
            complete_bootstrap_err_msg += "Please complete bootstrapping for "
            if len(dbs_to_be_bootstrapped) == 1:
                complete_bootstrap_err_msg += "database "
            else:
                complete_bootstrap_err_msg += "databases "
            complete_bootstrap_err_msg += "`{}`. ".format(','.join(dbs_to_be_bootstrapped))

        if dbs_not_to_be_bootstrapped and len(dbs_not_to_be_bootstrapped) != 0:
            complete_bootstrap_err_msg += "Please complete db schema creation for "
            if len(dbs_not_to_be_bootstrapped) == 1:
                complete_bootstrap_err_msg += "database "
            else:
                complete_bootstrap_err_msg += "databases "
            complete_bootstrap_err_msg += "`{}` on target cluster.".format(
                                                        ','.join(dbs_not_to_be_bootstrapped))

        return complete_bootstrap_err_msg

    def get_xcluster_error_msg(self, err, command, dbs_to_be_bootstrapped=None,
                                          dbs_not_to_be_bootstrapped=None,
                                          replication_id=None):
        # variable naming format: <<err_type>_<yb_admin/ybd>_err
        # Map format: <err present in yb_admin stderr>: <ybd err to be displayed>
        yb_admin_err_msg_to_ybd_err_msg = dict()
        if command == "create_checkpoint":
            empty_db_yb_admin_err = "Database should have at least one table in " +\
                "order to be part of xCluster replication"
            empty_db_ybd_err = "couldn't create xcluster checkpoint. " +\
                                "All provided databases should have atleast 1 table in them."
            rep_id_in_use_yb_admin_err = "xClusterOutboundReplicationGroup " +\
                                    "{} already exists".format(replication_id)
            rep_id_in_use_ybd_err = "xCluster Replication with replication ID: " +\
                    "'{}' already exists. Please use another replication ID".format(replication_id)
            yb_admin_err_msg_to_ybd_err_msg = {
                empty_db_yb_admin_err: empty_db_ybd_err,
                rep_id_in_use_yb_admin_err: rep_id_in_use_ybd_err,
            }
        elif command == "add_to_checkpoint":
            empty_db_yb_admin_err = "Database should have at least one table in " +\
                    "order to be part of xCluster replication"
            empty_db_ybd_err = "couldn't add to xcluster checkpoint. " +\
                                "Provided database should have atleast 1 table in it."
            yb_admin_err_msg_to_ybd_err_msg = {
                empty_db_yb_admin_err: empty_db_ybd_err,
            }
        elif command == "set_up":
            db_not_found_yb_admin_err = "YSQL keyspace name not found"
            object_not_found_yb_admin_err = "Could not find matching"

            complete_bootstrap_err_msg = self.get_complete_bootstrap_msg(dbs_to_be_bootstrapped,
                                                                         dbs_not_to_be_bootstrapped)

            db_not_found_ybd_err = "The database(s) is missing from the target cluster. " +\
                                                complete_bootstrap_err_msg
            object_not_found_ybd_err = "All of the SQL Objects are not present in the " +\
                                "target database. " + complete_bootstrap_err_msg
            yb_admin_err_msg_to_ybd_err_msg = {
                db_not_found_yb_admin_err: db_not_found_ybd_err,
                object_not_found_yb_admin_err: object_not_found_ybd_err,
            }
        elif command == "add_to_replication":
            db_not_found_yb_admin_err = "YSQL keyspace name not found"
            object_not_found_yb_admin_err = "Could not find matching"
            db_already_replicated_yb_admin_err = "xCluster ReplicationGroup already contains " +\
                                    "all requested tables"
            xcluster_not_set_up_yb_admin_err = "xCluster replication must already be setup in" +\
                 "order to add a new namespace to the target."

            complete_bootstrap_err_msg = self.get_complete_bootstrap_msg(dbs_to_be_bootstrapped,
                                                                         dbs_not_to_be_bootstrapped)

            db_not_found_ybd_err = "The database is missing from the target cluster. " +\
                                                complete_bootstrap_err_msg
            object_not_found_ybd_err = "All of the SQL Objects are not present in the " +\
                                "target database. " + complete_bootstrap_err_msg
            db_already_replicated_ybd_err = "The database is already added to the replication."
            xcluster_not_set_up_ybd_err = "xCluster replication is not set up. Please set up " +\
                "the replication first using the yugabyted xcluster set_up command."
            yb_admin_err_msg_to_ybd_err_msg = {
                db_not_found_yb_admin_err: db_not_found_ybd_err,
                object_not_found_yb_admin_err: object_not_found_ybd_err,
                db_already_replicated_yb_admin_err: db_already_replicated_ybd_err,
                xcluster_not_set_up_yb_admin_err: xcluster_not_set_up_ybd_err,
            }

        for yb_admin_err, ybd_err in yb_admin_err_msg_to_ybd_err_msg.items():
            if yb_admin_err in err:
                return ybd_err

        return ""

    def _extract_non_pii_failure_reason(self, error_msg):
        """Extract non-PII failure reason from error message.

        Args:
            error_msg: Error message that may contain PII (database names, etc.)

        Returns:
            Generic failure reason without PII
        """
        if not error_msg:
            return None

        error_lower = error_msg.lower()

        # Map error patterns to generic failure reasons
        if ("database is missing" in error_lower or
                "keyspace name not found" in error_lower):
            return "Database(s) missing from target"
        elif ("objects are not present" in error_lower or
                "could not find matching" in error_lower):
            return "DB Object(s) missing in target"
        elif ("already added to the replication" in error_lower or
                "already contains" in error_lower):
            return "Already replicated"
        elif ("should have at least one table" in error_lower or
                "should have atleast 1 table" in error_lower):
            return "Empty Database(s)"
        elif ("replication with replication id" in error_lower and
                "already exists" in error_lower):
            return "Replication id exists"
        elif "xcluster replication is not set up" in error_lower:
            return "XCluster not set up"
        elif "couldn't create xcluster checkpoint" in error_lower:
            return "Create Checkpoint Failed"
        elif "couldn't add to xcluster checkpoint" in error_lower:
            return "Add to Checkpoint Failed"
        elif "xcluster set-up not successful" in error_lower:
            return "Set Up Failed"
        elif "add to xcluster replication not successful" in error_lower:
            return "Add to Replication Failed"
        elif ("couldn't delete the replication" in error_lower or
                "couldn't delete database from xcluster replication" in
                error_lower):
            return "Delete Failed"
        elif "bootstrap" in error_lower:
            return "Bootstrap required"
        elif "replication group already exists" in error_lower or "already exists" in error_lower:
            return "Replication id exists"
        else:
            return "unknown_error"

    def _extract_failure_reasons_from_errors(self, error_dict, default_reason):
        """Extract and combine non-PII failure reasons from multiple errors.

        Args:
            error_dict: Dictionary mapping database/identifier to error message
            default_reason: Default reason to use if no valid reasons extracted

        Returns:
            Combined failure reason string (comma-separated unique reasons)
        """
        if not error_dict:
            return default_reason

        # If only one error, use _extract_non_pii_failure_reason directly
        if len(error_dict) == 1:
            error_msg = list(error_dict.values())[0]
            reason = self._extract_non_pii_failure_reason(error_msg)
            return reason if reason else default_reason

        # For multiple errors, extract unique reasons
        failure_reasons = set()
        for error_msg in error_dict.values():
            reason = self._extract_non_pii_failure_reason(error_msg)
            if reason:
                failure_reasons.add(reason)

        return ','.join(sorted(failure_reasons)) if failure_reasons else default_reason

    # Checkpoint xcluster replication
    def xcluster_create_checkpoint(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addrs = self.configs.saved_data.get("current_masters")
        if master_addrs == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        replication_id = self.configs.temp_data.get("xcluster_replication_id")
        databases = self.configs.temp_data.get("xcluster_databases")
        automatic_mode = self.configs.temp_data.get("xcluster_automatic_mode")

        start_time = time.time()
        out, err, ret_code = YBAdminProxy.create_checkpoint_xcluster(master_addrs, replication_id,
                                                              databases, automatic_mode)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        is_bootstrap_required = False

        if ret_code:
            err_msg = self.get_xcluster_error_msg(err, command = "create_checkpoint",
                                                    replication_id=replication_id)
            if not err_msg:
                err_msg = "couldn't create xcluster checkpoint."
            # Extract failure reason for callhome
            failure_reason = self._extract_non_pii_failure_reason(err_msg)
            # Send callhome event for xcluster create checkpoint (failure)
            database_count = len(databases.split(',')) if databases else 0
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    automatic_mode=automatic_mode,
                    time_taken=time_taken))
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT, timeout=5)
            Output.log_error_and_exit(Output.make_red("Error") +
                                ": {}".format(err_msg))
        else:
            bootstrap_required_msg = "Bootstrap is required"
            if bootstrap_required_msg in out:
                is_bootstrap_required = True

        status_details = []
        status_display_info = {}

        status_details = [
            (Output.make_yellow("Status"), "xCluster create checkpoint success.")
        ]

        # Send callhome event for xcluster create checkpoint (success)
        database_count = len(databases.split(',')) if databases else 0
        self.start_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT,
            lambda: self._send_xcluster_callhome(
                CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT,
                database_count=database_count,
                automatic_mode=automatic_mode,
                time_taken=time_taken))

        all_databases = databases.split(',')
        databases_to_be_bootstrapped = []
        databases_not_to_be_bootstrapped = []

        if is_bootstrap_required:
            regex = r'Perform a distributed Backup of database\(s\) \[(.*?)\] and ' + \
                r'Restore them on the target universe'
            match = re.search(regex, out)
            databases_to_be_bootstrapped = match.group(1).strip().split(',')

        databases_not_to_be_bootstrapped = [database for database in all_databases
                                           if database not in databases_to_be_bootstrapped]

        bootstrap_msg = ""
        schema_creation_msg = ""

        if is_bootstrap_required:
            if len(databases_to_be_bootstrapped) > 1:
                bootstrap_msg = "Bootstrap is required for databases " + \
                    "`{}`.".format(','.join(databases_to_be_bootstrapped))
            else:
                bootstrap_msg = "Bootstrap is required for database " + \
                    "`{}`.".format(databases_to_be_bootstrapped[0])

            bootstrap_help_msg = "For each database which requires bootstrap run the following " +\
                    "commands to perform a backup and restore.\n"
            bootstrap_help_msg += "# Run on source:\n"
            bootstrap_help_msg += "./yugabyted backup " +\
                    "--cloud_storage_uri {} ".format(Output.make_cyan(
                        "<AWS/GCP/local cloud storage uri>")) +\
                    " --database {}".format(Output.make_cyan("<database_name>")) +\
                    " --base_dir {}\n".format(Output.make_cyan("<base_dir of source node>"))
            bootstrap_help_msg += "# Run on target:\n"
            bootstrap_help_msg += "./yugabyted restore " +\
                    "--cloud_storage_uri {} ".format(Output.make_cyan(
                        "<AWS/GCP/local cloud storage uri>")) +\
                    " --database {}".format(Output.make_cyan("<database_name>")) +\
                    " --base_dir {}\n".format(Output.make_cyan("<base_dir of target node>"))

        if len(databases_not_to_be_bootstrapped) != 0:

            if len(databases_not_to_be_bootstrapped) > 1:
                schema_creation_msg = "Before running the xcluster setup command, Databases " +\
                        "`{}` and schema needs to be applied on the target cluster.".format(
                        ','.join(databases_not_to_be_bootstrapped))
            else:
                schema_creation_msg = "Before running the xcluster setup command, Database " +\
                        "`{}` and schema needs to be applied on the target cluster.".format(
                        databases_not_to_be_bootstrapped[0])


        if bootstrap_msg and schema_creation_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), bootstrap_msg),
                (Output.make_yellow(""), schema_creation_msg),
            ])
        elif bootstrap_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), bootstrap_msg),
            ])
        elif schema_creation_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), schema_creation_msg),
            ])

        Output.print_out(self.get_status_string_common(status_details, status_display_info))

        self.shutdown_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_CREATE_CHECKPOINT, timeout=5)
        if is_bootstrap_required:
            Output.print_out(bootstrap_help_msg)

    # Add to checkpoint xcluster replication
    def xcluster_add_to_checkpoint(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addrs = self.configs.saved_data.get("current_masters")
        if master_addrs == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        replication_id = self.configs.temp_data.get("xcluster_replication_id")
        databases = self.configs.temp_data.get("xcluster_databases")
        databases = databases.split(',')

        # Format for not_checkpointed_dbs map: <database>:<error_msg>
        not_checkpointed_dbs = dict()

        databases_to_be_bootstrapped = []
        databases_not_to_be_bootstrapped = []

        start_time = time.time()
        for database in databases:
            out, err, ret_code = YBAdminProxy.add_to_checkpoint_xcluster(master_addrs,
                                                                    replication_id, database)
            if ret_code:
                err_msg = self.get_xcluster_error_msg(err, command = "add_to_checkpoint",
                                                        replication_id=replication_id)
                if not err_msg:
                    err_msg = "couldn't add to xcluster checkpoint."

                not_checkpointed_dbs[database] = err_msg
                continue
            else:
                Output.log("Database {} added to checkpoint.".format(database))
                bootstrap_required_msg = "Bootstrap is required"
                if bootstrap_required_msg in out:
                    databases_to_be_bootstrapped.append(database)
                else:
                    databases_not_to_be_bootstrapped.append(database)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)

        status_details = []
        status_display_info = {}

        if len(not_checkpointed_dbs) == 0:
            status_details = [
               (Output.make_yellow("Status"), "xCluster add database(s) to checkpoint success.")
            ]
            # Send callhome event for xcluster add to checkpoint (success)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                    database_count=database_count,
                    time_taken=time_taken))
        elif len(not_checkpointed_dbs) == len(databases):
            # Extract failure reason for callhome
            failure_reason = self._extract_failure_reasons_from_errors(
                not_checkpointed_dbs, "all_databases_failed")

            # Send callhome event for xcluster add to checkpoint (failure)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    time_taken=time_taken))
            status_details = [
                (Output.make_yellow("Status"), "xCluster add database(s) to checkpoint failed."),
                (Output.make_yellow("Errors"), "Following are the errors for each database:")
            ]

            for database, error in not_checkpointed_dbs.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT, timeout=5)
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            sys.exit(1)
        else:
            # Extract failure reason for callhome (partial failure)
            failure_reason = self._extract_failure_reasons_from_errors(
                not_checkpointed_dbs, "partial_failure")

            # Send callhome event for xcluster add to checkpoint (partial failure)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    time_taken=time_taken))

            status_details = [
                (Output.make_yellow("Status"), "xCluster add database(s) to checkpoint " +
                                                        "partially succesful."),
                (Output.make_yellow("Errors"), "Following are the errors for database(s) " +
                                                        "failed to checkpoint:")
            ]

            for database, error in not_checkpointed_dbs.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )

        bootstrap_msg = ""
        schema_creation_msg = ""

        if len(databases_to_be_bootstrapped) != 0:
            if len(databases_to_be_bootstrapped) > 1:
                bootstrap_msg = "Bootstrap is required for databases " + \
                    "`{}`.".format(','.join(databases_to_be_bootstrapped))
            else:
                bootstrap_msg = "Bootstrap is required for database " + \
                    "`{}`.".format(databases_to_be_bootstrapped[0])

            bootstrap_help_msg = "For each database which requires bootstrap run the following " +\
                    "commands to perform a backup and restore.\n"
            bootstrap_help_msg += "# Run on source:\n"
            bootstrap_help_msg += "./yugabyted backup " +\
                    "--cloud_storage_uri {} ".format(Output.make_cyan(
                        "<AWS/GCP/local cloud storage uri>")) +\
                    " --database {}".format(Output.make_cyan("<database_name>")) +\
                    " --base_dir {}\n".format(Output.make_cyan("<base_dir of source node>"))
            bootstrap_help_msg += "# Run on target:\n"
            bootstrap_help_msg += "./yugabyted restore " +\
                    "--cloud_storage_uri {} ".format(Output.make_cyan(
                        "<AWS/GCP/local cloud storage uri>")) +\
                    " --database {}".format(Output.make_cyan("<database_name>")) +\
                    " --base_dir {}\n".format(Output.make_cyan("<base_dir of target node>"))

        if len(databases_not_to_be_bootstrapped) != 0:
            if len(databases_not_to_be_bootstrapped) > 1:
                schema_creation_msg = "Before running the xcluster setup command, Databases " +\
                        "`{}` and schema needs to be applied on the target cluster.".format(
                        ','.join(databases_not_to_be_bootstrapped))
            else:
                schema_creation_msg = "Before running the xcluster setup command, Database " +\
                        "`{}` and schema needs to be applied on the target cluster.".format(
                        databases_not_to_be_bootstrapped[0])


        if bootstrap_msg and schema_creation_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), bootstrap_msg),
                (Output.make_yellow(""), schema_creation_msg),
            ])
        elif bootstrap_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), bootstrap_msg),
            ])
        elif schema_creation_msg:
            status_details.extend([
                (Output.make_yellow("Bootstrapping"), schema_creation_msg),
            ])

        Output.print_out(self.get_status_string_common(status_details, status_display_info))

        # Shutdown callhome thread (reached end of function, not via early exit)
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_ADD_TO_CHECKPOINT, timeout=5)

        if len(databases_to_be_bootstrapped) != 0:
            Output.print_out(bootstrap_help_msg)

    # Set-up xcluster replication
    def xcluster_set_up(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addresses = self.configs.saved_data.get("current_masters")
        if master_addresses == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        replication_id = self.configs.temp_data.get("xcluster_replication_id")

        replication_id_with_dbs = self.get_databases_data_for_xcluster(replication_id)
        if len(replication_id_with_dbs) == 0:
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the " +
                                      "databases for replication_id {}.".format(replication_id))

        databases_list = [db.get("name") for db in replication_id_with_dbs[0].get("databases")]

        out, ret_code = YBAdminProxy.xcluster_is_bootstrap_required(master_addresses,
                                                                    replication_id,
                                                                    ','.join(databases_list))

        bootstrap_required_databases = []
        if ret_code:
            Output.log_error_and_exit(Output.make_red("Error") +
                            ": couldn't check if bootstrap is required or not")
        else:
            for database in databases_list:
                bootstrap_required_regex = re.compile(r'{} completed.\s*(.*?)\s*for'.format(
                                                                                        database))
                bootstrap_required_match = bootstrap_required_regex.search(out)
                if not bootstrap_required_match:
                    Output.log_error_and_exit("Couldn't find bootstrap requirement of database " +
                               "{}.".format(database))
                if bootstrap_required_match.group(1) == "Bootstrap is required":
                    bootstrap_required_databases.append(database)

        bootstrap_not_required_dbs = [database for database in databases_list \
                                      if database not in bootstrap_required_databases]

        Output.init_animation("Setting up xCluster replication....")
        start_time = time.time()
        out, err, ret_code = YBAdminProxy.set_up_xcluster(master_addresses, replication_id,
                                        self.configs.temp_data.get("xcluster_target_addresses"))
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        Output.update_animation("", Output.ANIMATION_STOP)
        status_details = []
        status_display_info = {}

        target_address = self.configs.temp_data.get("xcluster_target_addresses")
        target_address = target_address.split(',')[0].split(':')[0]
        database_count = len(databases_list) if databases_list else 0
        if ret_code == 0:
            status_details = [
                (Output.make_yellow("Status"), "xCluster set-up successful.")
            ]
            # Send callhome event for xcluster set up (success)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_SET_UP,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_SET_UP,
                    database_count=database_count,
                    target_address=target_address,
                    time_taken=time_taken))

            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_SET_UP, timeout=5)
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
        else:
            err_msg = self.get_xcluster_error_msg(err, command = "set_up",
                                        dbs_to_be_bootstrapped=bootstrap_required_databases,
                                        dbs_not_to_be_bootstrapped=bootstrap_not_required_dbs)
            if not err_msg:
                err_msg = "xCluster set-up not successful."
            # Extract failure reason for callhome
            failure_reason = self._extract_non_pii_failure_reason(err_msg)
            # Send callhome event for xcluster set up (failure)
            target_address = self.configs.temp_data.get("xcluster_target_addresses")
            database_count = len(databases_list) if databases_list else 0
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_SET_UP,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_SET_UP,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    target_address=target_address,
                    time_taken=time_taken))
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_SET_UP, timeout=5)
            Output.log_error_and_exit(Output.make_red("Error") +
                                ": {}".format(err_msg))

    # Set-up xcluster replication
    def xcluster_add_to_replication(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addresses = self.configs.saved_data.get("current_masters")
        if master_addresses == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        replication_id = self.configs.temp_data.get("xcluster_replication_id")
        databases = self.configs.temp_data.get("xcluster_databases")
        databases = databases.split(',')

        dbs_failed_to_add = dict()

        Output.init_animation("Adding database(s) to xCluster replication....")

        start_time = time.time()
        for database in databases:
            out, ret_code = YBAdminProxy.xcluster_is_bootstrap_required(master_addresses,
                                                                        replication_id,
                                                                        database)
            is_bootstrap_required = False
            if ret_code:
                Output.log(Output.make_red("Error") + ": couldn't check if bootstrap is " +
                           "required or not for database {}".format(database))
            else:
                bootstrap_check_msg = "Bootstrap is required for setting up xCluster replication"
                if bootstrap_check_msg in out:
                    is_bootstrap_required = True

            out, err, ret_code = YBAdminProxy.add_to_replication_xcluster(master_addresses,
                                            replication_id, database,
                                            self.configs.temp_data.get("xcluster_target_addresses"))

            if ret_code:
                err_msg = None
                if is_bootstrap_required:
                    err_msg = self.get_xcluster_error_msg(err, command = "add_to_replication",
                                            dbs_to_be_bootstrapped=[database],
                                            dbs_not_to_be_bootstrapped=[])
                else:
                    err_msg = self.get_xcluster_error_msg(err, command = "add_to_replication",
                                            dbs_to_be_bootstrapped=[],
                                            dbs_not_to_be_bootstrapped=[database])
                if not err_msg:
                    err_msg = "Add to xCluster replication not successful."

                dbs_failed_to_add[database] = err_msg
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)

        Output.update_animation("", Output.ANIMATION_STOP)
        status_details = []
        status_display_info = {}

        target_address = self.configs.temp_data.get("xcluster_target_addresses")

        if len(dbs_failed_to_add) == 0:
            status_details = [
               (Output.make_yellow("Status"), "Add database(s) to xCluster replication successful.")
            ]
            # Send callhome event for xcluster add to replication (success)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                    database_count=database_count,
                    target_address=target_address,
                    time_taken=time_taken))
        elif len(dbs_failed_to_add) == len(databases):
            # Extract non-PII failure reason
            failure_reason = self._extract_failure_reasons_from_errors(
                dbs_failed_to_add, "all_databases_failed")

            # Send callhome event for xcluster add to replication (failure)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                    replication_id, success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    target_address=target_address,
                    time_taken=time_taken))

            status_details = [
                (Output.make_yellow("Status"), "Add database(s) to xCluster replication failed."),
                (Output.make_yellow("Errors"), "Following are the errors for each database:")
            ]

            for database, error in dbs_failed_to_add.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION, timeout=5)
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            sys.exit(1)
        else:
            # Extract non-PII failure reason for partial failure
            failure_reason = self._extract_failure_reasons_from_errors(
                dbs_failed_to_add, "partial_failure")

            # Send callhome event for xcluster add to replication (partial failure)
            database_count = len(databases)
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    target_address=target_address,
                    time_taken=time_taken))

            status_details = [
                (Output.make_yellow("Status"), "xCluster add database(s) to replication " +
                                                        "partially succesful."),
                (Output.make_yellow("Errors"), "Following are the errors for database(s) " +
                                                        "failed to checkpoint:")
            ]

            for database, error in dbs_failed_to_add.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )


        Output.print_out(self.get_status_string_common(status_details, status_display_info))

        # Shutdown callhome thread (reached end of function, not via early exit)
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_ADD_TO_REPLICATION, timeout=5)

    # Show Outbound and Inbound replications status
    def xcluster_status(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addresses, leader_master = self.get_current_masters_and_leader_from_api(
            self.advertise_ip())
        if master_addresses == '' or leader_master == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        replication_id = self.configs.temp_data.get("xcluster_replication_id")

        xcluster_info = self.get_xcluster_info(leader_master)
        if len(xcluster_info) == 0:
            Output.log_error_and_exit("Couldn't fetch xcluster info.")

        outbound_replication_status = self.get_outbound_xcluster_replication_statuses(xcluster_info,
                              replication_id if replication_id != "" else None)

        inbound_replication_status = self.get_inbound_xcluster_replication_statuses(xcluster_info,
                              master_addresses, replication_id if replication_id != "" else None)

        if outbound_replication_status:
            Output.print_out(Output.make_yellow("Outbound xCluster Replications:"))
            Output.print_out(outbound_replication_status)
        else:
            if replication_id:
                Output.print_out("No Outbound xCluster replications found for this replication_id.")
            else:
                Output.print_out("No Outbound xCluster replications found for this cluster.")

        if inbound_replication_status:
            Output.print_out(Output.make_yellow("Inbound xCluster Replications:"))
            Output.print_out(inbound_replication_status)
        else:
            if replication_id:
                Output.print_out("No Inbound xCluster replications found for this replication_id.")
            else:
                Output.print_out("No Inbound xCluster replications found for this cluster.")

        if not inbound_replication_status and not outbound_replication_status:
            Output.log_error_and_exit("No inbound and Outbound replication groups.")

    # Delete xcluster replication
    def xcluster_delete_replication(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addresses = self.configs.saved_data.get("current_masters")
        if master_addresses == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        is_target_available = True
        target_master_addresses = self.get_current_masters_from_api(
                                        self.configs.temp_data.get("xcluster_target_addresses"))
        if target_master_addresses == '':
            is_target_available = False

        replication_id = self.configs.temp_data.get("xcluster_replication_id")

        start_time = time.time()
        err, ret_code = YBAdminProxy.drop_xcluster_replication(master_addresses,
                                        replication_id, target_addresses=target_master_addresses)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)

        if ret_code:
            err_msg = self.get_xcluster_error_msg(err, command = "delete_replication",
                                                  replication_id=replication_id)
            if not err_msg:
                err_msg = "Couldn't delete the replication."
            # Extract failure reason for callhome
            failure_reason = self._extract_non_pii_failure_reason(err_msg)
            # Send callhome event for xcluster delete replication (failure)
            target_address = self.configs.temp_data.get("xcluster_target_addresses")
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION,
                    success=False,
                    failure_reason=failure_reason,
                    target_address=target_address,
                    is_target_available=is_target_available,
                    time_taken=time_taken))
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION, timeout=5)
            Output.log_error_and_exit(Output.make_red("Error") +
                                ": {}".format(err_msg))
        status_details = []
        status_display_info = {}

        # Send callhome event for xcluster delete replication (success)
        target_address = self.configs.temp_data.get("xcluster_target_addresses")
        self.start_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION,
            lambda: self._send_xcluster_callhome(
                CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION,
                target_address=target_address,
                is_target_available=is_target_available,
                time_taken=time_taken))

        if is_target_available:
            status_details = [
                (Output.make_yellow("Status"), "xCluster successfully deleted.")
            ]
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
        else:
            msg = Output.make_yellow(Output.ANIMATION_WARNING + " WARNING") + ": Couldn't reach "+\
                        "target cluster. Deleting the replication on source cluster."

            status_details = [
                (Output.make_yellow("Status"), "xCluster successfully deleted on source."),
            ]
            Output.print_out(msg)
            Output.print_out(self.get_status_string_common(status_details, status_display_info))

            msg = Output.make_cyan("NOTE") + ": After restarting the target, please run the " +\
                                            "following command on a target node to delete " +\
                                            "replication from target cluster:\n"
            msg += "\n./yugabyted configure admin_operation " + \
                "--base_dir {} ".format(Output.make_cyan("<base_dir_for_the_target_node>")) + \
                "--command {}".format(
                    Output.make_cyan("delete_universe_replication <replication_id>"))
            Output.print_out(msg)

        # Shutdown callhome thread for success case
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_DELETE_REPLICATION, timeout=5)

    # Delete database from xcluster replication
    def xcluster_remove_database_from_replication(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        master_addresses = self.configs.saved_data.get("current_masters")
        if master_addresses == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        target_master_addresses = self.get_current_masters_from_api(
                                        self.configs.temp_data.get("xcluster_target_addresses"))

        replication_id = self.configs.temp_data.get("xcluster_replication_id")
        databases = self.configs.temp_data.get("xcluster_databases")

        not_deleted_dbs = dict()

        start_time = time.time()
        for database in databases.split(','):
            err, ret_code = YBAdminProxy.delete_from_xcluster_replication(master_addresses,
                                replication_id, database, target_addresses=target_master_addresses)
            if ret_code:
                err_msg = self.get_xcluster_error_msg(err,
                                                      command = "remove_database_from_replication",
                                                      replication_id=replication_id)
                if not err_msg:
                    err_msg = "couldn't delete database from xcluster replication."

                not_deleted_dbs[database] = err_msg
                continue
            else:
                if target_master_addresses == '':
                    Output.log("Database {} deleted from replication ".format(database) +
                               "from source only.")
                else:
                    Output.log("Database {} deleted from replication.".format(database))
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)

        status_details = []
        status_display_info = {}

        if len(not_deleted_dbs) == 0:
            status_details = [
               (Output.make_yellow("Status"), "xCluster delete database(s) from " +
                            "replication success.")
            ]
            # Send callhome event for xcluster remove database from replication (success)
            database_count = len(databases.split(',')) if databases else 0
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                    database_count=database_count,
                    time_taken=time_taken))
        elif len(not_deleted_dbs) == len(databases):
            # Extract failure reason for callhome
            failure_reason = self._extract_failure_reasons_from_errors(
                not_deleted_dbs, "all_databases_failed")

            # Send callhome event for xcluster remove database from replication (failure)
            database_count = len(databases.split(',')) if databases else 0
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                    replication_id, success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    time_taken=time_taken))

            status_details = [
                (Output.make_yellow("Status"), "xCluster delete database(s) from " +
                                                    "replication failed."),
                (Output.make_yellow("Errors"), "Following are the errors for each database:")
            ]

            for database, error in not_deleted_dbs.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION, timeout=5)
            Output.print_out(self.get_status_string_common(status_details, status_display_info))
            sys.exit(1)
        else:
            # Extract failure reason for callhome (partial failure)
            failure_reason = self._extract_failure_reasons_from_errors(
                not_deleted_dbs, "partial_failure")

            # Send callhome event for xcluster remove database from replication (partial failure)
            database_count = len(databases.split(',')) if databases else 0
            self.start_callhome_thread(
                CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                lambda: self._send_xcluster_callhome(
                    CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION,
                    success=False,
                    failure_reason=failure_reason,
                    database_count=database_count,
                    time_taken=time_taken))

            status_details = [
                (Output.make_yellow("Status"), "xCluster delete database(s) from " +
                                                    "replication partially succesful."),
                (Output.make_yellow("Errors"), "Following are the errors for database(s) " +
                                                        "failed to get delete:")
            ]

            for database, error in not_deleted_dbs.items():
                status_details.extend(
                    [(Output.make_yellow(""), "{}: {}".format(database, error))]
                )

        Output.print_out(self.get_status_string_common(status_details, status_display_info))

        # Shutdown callhome thread (reached end of function, not via early exit)
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_XCLUSTER_REMOVE_DATABASE_FROM_REPLICATION, timeout=5)

    # Configuring the primary cluster data placement policy
    def configure_data_placement(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use `yugabyted " +
                                      "configure data_placement` command from a read replica " +
                                      "node. Please use `yugabyted configure_read_replica " +
                                      "[new/modify/delete]` instead.")

        fault_tolerance = self.configs.temp_data.get("fault_tolerance")
        replication_factor = self.configs.temp_data.get("replication_factor")

        if fault_tolerance == "cloud":
            Output.log_error_and_exit("Cloud based fault tolerance is not supported yet.")

        current_masters_with_port_list = self.get_all_masters()
        current_masters_without_port_list = [ urlparse("//" + master).hostname for
                                             master in current_masters_with_port_list ]
        leader_master = urlparse("//" + self.get_leader_master()).hostname
        leader_master_url_address = get_url_from_ip(leader_master)
        master_addr = "{}:{}".format(leader_master_url_address,
                                        self.configs.saved_data.get("master_rpc_port"))

        leader_master_http_endpoint = "{}:{}".format(leader_master_url_address,
                        self.configs.saved_data.get("master_webserver_port"))
        placement_uuid = self.configs.saved_data.get("placement_uuid")

        all_tserver_info = self.get_all_tserver_info(leader_master_http_endpoint)
        placement_locations = self.get_all_nodes_locations(all_tserver_info, placement_uuid)

        if len(placement_locations) < int(replication_factor):
            Output.print_and_log(msg = Output.make_red("ERROR") + ": not enough nodes to set-up " +
                                "a RF-{} cluster".format(replication_factor), level=logging.ERROR)
            sys.exit(1)

        new_masters = self.get_new_valid_masters(current_masters_without_port_list,
                leader_master, placement_locations)

        masters_to_remove = [ master for master in current_masters_without_port_list
                             if master not in new_masters ]
        masters_to_add = [ master for master in new_masters
                          if master not in current_masters_without_port_list ]

        if not masters_to_remove and not masters_to_add:
            Output.log("Desired fault tolerance is already setup. No Masters to move.")
        else :
            track_masters_to_remove = masters_to_remove[:]
            track_masters_to_add = masters_to_add[:]
            if masters_to_remove and masters_to_add:
                for i in range(len(masters_to_remove)):
                    self.replace_master(master_addr, masters_to_remove[i], masters_to_add[i])
                    track_masters_to_remove.remove(masters_to_remove[i])
                    track_masters_to_add.remove(masters_to_add[i])

            if len(track_masters_to_add) != 0:
                for master in track_masters_to_add:
                    self.add_master_for_data_placement(master_addr, master)

            if len(track_masters_to_remove) != 0:
                for master in track_masters_to_remove:
                    self.remove_master_for_data_placement(master_addr, master)

        current_masters_with_port_list = self.get_all_masters()
        if self.configs.temp_data.get("constraint_value"):
            constraint_value = self.configs.temp_data.get("constraint_value")
            placement_info,priority_info = self.parse_constraint_value(constraint_value)
        else:
            placement_info = []
            priority_info = []
            for master in current_masters_with_port_list:
                placement_info.append(placement_locations[urlparse("//" + master).hostname])
            placement_info = ",".join(placement_info)

        if not self.is_placement_constraint_valid_length(placement_info):
            Output.log_error_and_exit(Output.make_red("FAILED") + ": Invalid number of " +
                                      "placement constraint specified.")

        if not self.is_placement_constraint_valid_values(placement_locations, placement_info):
            Output.log_error_and_exit(Output.make_red("FAILED") + ": Invalid values provided " +
                                      "for placement constraint.")

        master_addresses = ",".join(current_masters_with_port_list)

        if not YBAdminProxy.modify_placement_info(master_addresses, placement_info,
                                                  placement_uuid, replication_factor):
            Output.log_error_and_exit("FAILED: Setting of placement info of masters")
        else:
            Output.log("Configure step was successful.")

        if priority_info:
            priority_info = list(set(priority_info))
            if not YBAdminProxy.set_preferred_zones(master_addresses, priority_info):
                Output.log_error_and_exit(Output.make_red("FAILED") +
                                          ": Unsuccessful in setting preference for zones")
            else:
                Output.log("Successful in setting preference for zones.")

        status_details = self.get_configure_status_details(placement_locations,
                                                           new_masters, priority_info)

        Output.print_out(self.get_status_string_common(status_details[0], status_details[1]))

    # Generate the node server certificates
    def cert_generate_server_certs(self):
        if self.check_openssl():
            Output.log_error_and_exit(Output.make_red("Error") + ": openssl not installed. " +
                "Can't create certificates.")

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use `yugabyted cert " +
                                      "generate_server_certs` command from a read replica node. " +
                                      "Please use a primary cluster node to generate certs.")

        gen_certs_dir = self.configs.saved_data.get("gen_certs_dir")
        root_certs_dir = os.path.join(gen_certs_dir, "root-ca")

        status_details = []
        if not self.generate_ca_certs(root_certs_dir=root_certs_dir):
            status_details = [
                (Output.make_yellow("Status"), "Cert generation failed. Please check the logs."),
            ]
            Output.print_out(self.get_status_string_common(status_details))
            sys.exit(1)

        hostnames = self.configs.temp_data.get("hostnames")
        hostnames = hostnames.split(',')

        generated_certs_hostnames = self.generate_node_server_certs(hostnames=hostnames,
                                                                    gen_certs_dir=gen_certs_dir)
        if len(generated_certs_hostnames):
            status_details = [
                (Output.make_yellow("Status"), "Server nodes certs generation for hostnames " +
                                "{} successful.".format(",".join(generated_certs_hostnames))),
                (Output.make_yellow("Server node certs path"),
                        "Certs can be found at: {}".format(gen_certs_dir+"/<hostname>")),
            ]
        else:
            status_details = [
                (Output.make_yellow("Status"), "No certs generated. All hostnames already have " +
                                               "certs generated."),
                (Output.make_yellow("Server node certs path"),
                        "Certs can be found at: {}".format(gen_certs_dir+"/<hostname>")),
            ]

        Output.print_out(self.get_status_string_common(status_details))

    # Generate the root certificates
    def generate_ca_certs(self, root_certs_dir):
        generate_certs = False

        if os.path.isdir(root_certs_dir):
            if self.check_root_cert_files(root_certs_dir):
                Output.log("Found an existing root certs directory {}.".format(root_certs_dir) +
                    " Found appropriate files. Using the existing root certs.")
            else:
                Output.print_and_log(Output.make_yellow(Output.ANIMATION_WARNING + " WARNING:") +
                    " Found an existing root certs directory {}.".format(root_certs_dir) +
                    " Appropriate files not found. Removing these and creating new root certs." +
                    " Please recreate any server certs created with previous root certs.")
                shutil.rmtree(root_certs_dir)

                generate_certs = True
        else:
            generate_certs = True

        if generate_certs:
            Output.log("Creating root certs...")
            if not os.path.isdir(root_certs_dir):
                os.makedirs(root_certs_dir)

            status = OpenSSLProxy.generate_root_ca_certs(root_certs_dir=root_certs_dir)

            if not status:
                return False

        return True

    # Generate node server certififcates
    def generate_node_server_certs(self, hostnames, gen_certs_dir):
        root_certs_dir = os.path.join(gen_certs_dir, "root-ca")

        existing_certs_hostnames = []
        generated_certs_hostnames = []
        # Generate certs for each server node
        for hostname in hostnames:
            # Check whether the hostname is already present in the certs database or not.
            if self.check_hostname_in_root_ca_database(hostname=hostname,
                                                       root_certs_dir=root_certs_dir):
                existing_certs_hostnames.append(hostname)
                continue

            # Check whether the hostname cert folder is present or not
            node_certs_dir = os.path.join(gen_certs_dir, hostname)
            if os.path.isdir(node_certs_dir):
                Output.log("Found an existing cert directory {} for ".format(node_certs_dir) +
                    "hostname {}. But no data entry found in ".format(hostname) +
                    "root-ca certs database. Removing...")
                shutil.rmtree(node_certs_dir)

            if self.configs.saved_data.get("dns_enabled"):
                hostname_type = "DNS"
            else:
                hostname_type = "IP"
            status = OpenSSLProxy.generate_node_server_certs(root_certs_dir=root_certs_dir,
                hostname=hostname, server_cert_dir=node_certs_dir, hostname_type=hostname_type)

            if not status:
                status_details = [
                    (Output.make_yellow("Status"), "Server node cert generation failed for " +
                        "hostname {}. Please check the logs.".format(hostname)),
                ]
                Output.print_out(self.get_status_string_common(status_details))
                exit(1)
            else:
                generated_certs_hostnames.append(hostname)

        if len(existing_certs_hostnames):
            Output.print_and_log(Output.make_yellow(Output.ANIMATION_WARNING + " WARNING:") +
                " Hostnames {} ".format(Output.make_green(",".join(existing_certs_hostnames))) +
                "already have their certifcates generated according to root-ca certs database " +
                "index. Please use the previously generated certs. Skipping these hostnames....")


        return generated_certs_hostnames

    # Check if all root certificate files are present in a path or not
    def check_root_cert_files(self, path):
        root_certs_files = ["ca.crt", "ca.key", "ca.conf", "index.txt",
            "index.txt.attr", "serial.txt"]

        return check_files_in_path(path, root_certs_files)

    # Check if a hostname is present in the root-ca database
    def check_hostname_in_root_ca_database(self, hostname, root_certs_dir):
        root_ca_database_file = os.path.join(root_certs_dir, "index.txt")

        with open(root_ca_database_file, 'r') as certs_database_file:
            certs_database = certs_database_file.read()
            if hostname in certs_database:
                return True

        return False

    # Use the masters to check if the masters has the key with <key_id>.
    def is_key_in_all_masters(self, key_id):
        masters = ",".join(self.get_all_masters())

        # IPs of the masters which are currently in the cluster
        master_ips = [urlparse("//" + master).hostname for master in masters.split(',')]

        # Masters which have the key with id as key_id
        masters_with_key = YBAdminProxy.check_key_in_masters(master_addrs=masters, key_id=key_id)
        for i, master in enumerate(masters_with_key):
            master = urlparse("//" + master.split()[1]).hostname
            masters_with_key[i] = master

        ret = True
        if len(masters_with_key) < len(master_ips):
            ret = False

        for master in master_ips:
            if master not in masters_with_key:
                ret = False

        return ret

    # Use the masters to check if encryption at rest is enabled.
    def is_encryption_at_rest_enabled(self):
        masters = self.configs.saved_data.get("current_masters")
        result = YBAdminProxy.check_encryption(master_addrs=masters)

        if result is None:
            return None

        if "ENABLED" in result:
            return True
        else:
            return False

    # Configuring encryption at rest
    def configure_encrypt_at_rest(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use `yugabyted " +
                                      "configure encrypt_at_rest` command from a read replica " +
                                      "node. Please use a primary cluster node to enable EAR.")

        status_details = []
        has_errors = False

        # Enable encryption at rest.
        if self.configs.temp_data.get("enable_encrypt_at_rest"):
            Output.log("Trying to enable encryption at rest.")

            # Check whether the encryption at rest is already enabled.
            encryption_enabled = self.is_encryption_at_rest_enabled()
            if encryption_enabled is None:
                Output.log_error_and_exit("Error checking status of encryption-at-rest.")
            elif encryption_enabled:
                Output.log("Encryption at rest already enabled.")
                has_errors = True
                status_details = [
                    (Output.make_yellow("Status"), "Encryption at rest already enabled.")
                ]

            # Setting the key directory and name
            key_dir = os.path.join(self.configs.saved_data.get("data_dir"), "encrypt-keys")
            keyname = "encrypt_at_rest"

            if not has_errors:
                # Check openssl is installed in the machine or not.
                if self.check_openssl():
                    Output.log_error_and_exit(Output.make_red("Error") + ": openssl not " +
                        "installed. Can't create universe key for encryption.")

                # Generate the universe key.
                status = OpenSSLProxy.generate_key(key_dir=key_dir, keyname=keyname)

                if not status:
                    Output.log(Output.make_red("Error") + ": Universe key generation failed for " +
                            "encryption at rest.")
                    has_errors = True
                    status_details = [
                        (Output.make_yellow("Status"), "Universe key generation failed for " +
                            "enabling encryption at rest. Please check the logs.")
                    ]
                else:
                    masters = ",".join(self.get_all_masters())
                    key_id = str(uuid.uuid4())
                    key_path = key_dir + "/" + keyname + ".key"

                    # Copy the generated universe key to all the masters
                    if not has_errors and not YBAdminProxy.copy_key_to_masters(
                            master_addrs=masters, key_id=key_id, key_path=key_path):
                        Output.log(Output.make_red("Error") + ": cannot copy the " +
                            "generated key to the masters.")
                        has_errors = True
                        status_details = [
                            (Output.make_yellow("Status"), "Encryption at rest failed. " +
                                "Please check the logs."),
                        ]

                    # Check whether all masters have the key.
                    if not has_errors:
                        if not self.is_key_in_all_masters(key_id=key_id):
                            Output.log(Output.make_red("Error") + ": Universe key " +
                                "{} not found in all masters.".format(key_id))
                            has_errors = True
                            status_details = [
                                (Output.make_yellow("Status"), "Encryption at rest failed. " +
                                    "Please check the logs."),
                            ]
                        else:
                            Output.log("Copied key {} to all the masters".format(key_id))

                    # Enable encryption at rest.
                    if not has_errors and not YBAdminProxy.enable_encryption_using_key(
                            master_addrs=masters, key_id=key_id):
                        Output.log(Output.make_red("Error") + ": cannot enable " +
                            "encryption at rest.")
                        has_errors = True
                        status_details = [
                            (Output.make_yellow("Status"), "Encryption at rest failed. " +
                                "Please check the logs."),
                        ]

                    # Check whether the encryption at rest has been enabled or not.
                    if not has_errors:
                        encryption_enabled = self.is_encryption_at_rest_enabled()
                        if encryption_enabled is None:
                            Output.log_error_and_exit("Error checking status of " +
                                "encryption-at-rest.")
                        elif not encryption_enabled:
                            Output.log(Output.make_red("Error") + ": cannot enable " +
                                "encryption at rest.")
                            has_errors = True
                            status_details = [
                                (Output.make_yellow("Status"), "Encryption at rest failed. " +
                                    "Please check the logs."),
                            ]
                        else:
                            out = "Encryption at rest enabled with key: {}".format(key_id)
                            Output.log(out)
                            status_details = [
                                (Output.make_yellow("Status"), out),
                                (Output.make_yellow("Key Location"), f"The Encryption key is " +
                                    f"stored at {key_path}. Moving the encryption key to a " +
                                    "secure location is strongly advised. "),
                            ]
        # Disabling encryption at rest
        else:
            Output.log("Trying to disable encryption at rest.")

            # Check whether encryption at rest is already disabled.
            encryption_enabled = self.is_encryption_at_rest_enabled()
            if encryption_enabled is None:
                Output.log_error_and_exit("Error checking status of encryption-at-rest.")
            elif not encryption_enabled:
                Output.log("Encryption at rest is already disabled.")
                has_errors = True
                status_details = [
                    (Output.make_yellow("Status"), "Encryption at rest is already disabled.")
                ]

            masters = ",".join(self.get_all_masters())

            # Disable encryption at rest
            if not has_errors and not YBAdminProxy.disable_encryption(master_addrs=masters):
                Output.log(Output.make_red("Error") + ": cannot disable encryption at rest.")
                has_errors = True
                status_details = [
                    (Output.make_yellow("Status"), "Disabling encryption at rest failed. " +
                        "Please check the logs."),
                ]

            # Check whether encryption at rest has been disabled or not.
            if not has_errors:
                encryption_enabled = self.is_encryption_at_rest_enabled()
                if encryption_enabled is None:
                    Output.log_error_and_exit("Error checking status of encryption-at-rest.")
                elif encryption_enabled:
                    Output.log(Output.make_red("Error") + ": cannot disable encryption at rest.")
                    has_errors = True
                    status_details = [
                        (Output.make_yellow("Status"), "Disabling encryption at rest failed. " +
                            "Please check the logs."),
                    ]
                else:
                    out = "Encryption at rest disabled."
                    Output.log(out)
                    status_details = [
                        (Output.make_yellow("Status"), out),
                    ]

        Output.print_out(self.get_status_string_common(status_details))

    # handle the admin operations passthrough to yb-admin command.
    def configure_admin_operation(self, timeout=10):

        master_addrs = self.configs.temp_data.get("admin_operation_master_addresses")
        if not master_addrs:
            master_addrs = ",".join(self.get_all_masters())
        command = self.configs.temp_data.get("admin_command")
        command_args = command.split(" ")
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs]
        for cmd_var in command_args:
            cmd += [cmd_var]

        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)

        status_details = {}
        status_details = [(Output.make_yellow("Admin operation"),
                           "Find the response below."),]
        Output.print_out(self.get_status_string_common(status_details))
        if 0 == ret_code:
          Output.print_out(out)
        else:
          Output.log(err)
          Output.print_out(Output.make_red(err))

    # Checks if the read replica cluster has been already configured or not.
    def is_read_replica_configured(self, cluster_config):
        if cluster_config and "replicationInfo" in cluster_config:
            if "readReplicas" in cluster_config["replicationInfo"]:
                return True

        return False

    # Configuring the read replica cluster data placement policy
    def configure_read_replica_new(self):
        # Check if a node is running or not
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        # Check if read replica cluster has been already configured?
        current_masters_with_port_list = self.get_all_masters()
        master_addresses = ",".join(current_masters_with_port_list)
        cluster_config = YBAdminProxy.get_cluster_config(master_addresses)
        if self.is_read_replica_configured(cluster_config):
            Output.print_and_log(Output.make_red("ERROR") + ": Read replica cluster has already " +
                                 "been configured. If you want to change the configuration " +
                                 "please use 'yugabyted configure_read_replica modify' command.")
            sys.exit(1)

        leader_master = urlparse("//" + self.get_leader_master()).hostname
        leader_master_http_endpoint = "{}:{}".format(get_url_from_ip(leader_master),
                        self.configs.saved_data.get("master_webserver_port"))

        all_tserver_info = self.get_all_tserver_info(leader_master_http_endpoint)

        placement_uuid = self.configs.saved_data.get("placement_uuid")
        # Use read replica placement UUID if yugabyted is not running in a read replica node.
        if not self.configs.saved_data.get("read_replica"):
            placement_uuid = [uuid for uuid in list(all_tserver_info.keys())
                              if uuid!=placement_uuid][0]

        placement_locations = self.get_all_nodes_locations(all_tserver_info, placement_uuid)

        rf_and_placement_policy_list = self.get_rr_rf_and_placement_constraint(placement_locations)
        rr_replication_factor = rf_and_placement_policy_list[0]
        rr_placement_constraint = rf_and_placement_policy_list[1]

        status_details = list()
        if not YBAdminProxy.set_read_replica_placement(master_addresses, rr_placement_constraint,
                                                  placement_uuid, rr_replication_factor):
            display_msg = "FAILED: Setting of cluster config of read replica cluster."
            status_details.extend([
                (Output.make_yellow("Status"), display_msg)
            ])
        else:
            display_msg = [
                "Configuration successful.",
                "The primary cluster will begin asynchronous replication with " +
                "the read replica cluster.",
            ]
            status_details.extend([
                (Output.make_yellow("Status"), display_msg[0]),
                (Output.make_yellow(""), display_msg[1]),
            ])

        Output.print_out(self.get_status_string_common(status_details))

    # Configuring the read replica cluster data placement policy
    def configure_read_replica_modify(self):
        # Check if a node is running or not
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        # Check if read replica cluster has been already configured?
        current_masters_with_port_list = self.get_all_masters()
        master_addresses = ",".join(current_masters_with_port_list)
        cluster_config = YBAdminProxy.get_cluster_config(master_addresses)
        if not self.is_read_replica_configured(cluster_config):
            Output.print_and_log(Output.make_red("ERROR") + ": Read replica cluster has not " +
                                 "been configured. Please set the read replica cluster " +
                                 "configuration using 'yugabyted configure_read_replica new' " +
                                 "command before modifying it.")
            sys.exit(1)

        leader_master = urlparse("//" + self.get_leader_master()).hostname
        leader_master_http_endpoint = "{}:{}".format(get_url_from_ip(leader_master),
                        self.configs.saved_data.get("master_webserver_port"))

        all_tserver_info = self.get_all_tserver_info(leader_master_http_endpoint)

        placement_uuid = self.configs.saved_data.get("placement_uuid")
        # Use read replica placement UUID if yugabyted is not running in a read replica node.
        if not self.configs.saved_data.get("read_replica"):
            placement_uuid = [uuid for uuid in list(all_tserver_info.keys())
                              if uuid!=placement_uuid][0]

        placement_locations = self.get_all_nodes_locations(all_tserver_info, placement_uuid)

        self.validate_new_config_for_rr_modify(cluster_config)

        rf_and_placement_policy_list = self.get_rr_rf_and_placement_constraint(placement_locations)
        rr_replication_factor = rf_and_placement_policy_list[0]
        rr_placement_constraint = rf_and_placement_policy_list[1]

        status_details = list()
        if not YBAdminProxy.modify_read_replica_placement(master_addresses, rr_placement_constraint,
                                                  placement_uuid, rr_replication_factor):
            display_msg = "FAILED: Modifying of cluster config of read replica cluster."
            status_details.extend([
                (Output.make_yellow("Status"), display_msg)
            ])
        else:
            display_msg = [
                "Configuration successful.",
                "The primary cluster will begin asynchronous replication with " +
                "the read replica cluster.",
            ]
            status_details.extend([
                (Output.make_yellow("Status"), display_msg[0]),
                (Output.make_yellow(""), display_msg[1]),
            ])

        Output.print_out(self.get_status_string_common(status_details))

    # Configuring the read replica cluster data placement policy
    def configure_read_replica_delete(self):
        # Check if a node is running or not
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        # Check if read replica cluster has been already configured?
        current_masters_with_port_list = self.get_all_masters()
        master_addresses = ",".join(current_masters_with_port_list)
        cluster_config = YBAdminProxy.get_cluster_config(master_addresses)
        if not self.is_read_replica_configured(cluster_config):
            Output.print_and_log(Output.make_red("ERROR") + ": Read replica cluster has not " +
                                 "been configured.")
            sys.exit(1)

        status_details = list()
        if not YBAdminProxy.delete_read_replica_placement(master_addresses):
            display_msg = "FAILED: Deletion of read replica cofiguration."
            status_details.extend([
                (Output.make_yellow("Status"), display_msg)
            ])
        else:
            status_details.extend([
                    (Output.make_yellow("Status"), "Configure delete step was successful."),
            ])

        Output.print_out(self.get_status_string_common(status_details))

    # Upgrade the ysql catalog for this node
    def upgrade_ysql_catalog(self):
        if not self.script.is_running():
            Output.log_error_and_exit(Output.make_red("ERROR") + ": No YugabyteDB node " +
                "is running in the data_dir {}".format(self.configs.saved_data.get("data_dir")))

        if self.configs.saved_data.get("read_replica"):
            Output.log_error_and_exit(Output.make_red("ERROR") + ": Cannot use `yugabyted " +
                                      "upgrade_ysql_catalog` command from a read replica node. " +
                                      "Please use a primary cluster node to upgrade the " +
                                      "YSQL catalog.")

        master_addresses = self.configs.saved_data.get("current_masters")
        if master_addresses == '':
            Output.log_error_and_exit(Output.make_red("Error") + ": cannot retreive the masters.")

        timeout = 300000
        if self.configs.temp_data.get("upgrade_timeout") is not None:
            timeout = self.configs.temp_data.get("upgrade_timeout")

        # Start the YSQL catalog upgrade
        Output.init_animation("YSQL Catalog upgrade in progress...")
        start_time = time.time()
        _, ret_code = YBAdminProxy.start_ysql_catalog_upgrade(
                master_addresses, timeout)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        success = (ret_code == 0)
        # Send callhome event for upgrade ysql catalog
        self.start_callhome_thread(
            CALLHOME_EVENT_UPGRADE_YSQL_CATALOG,
            lambda: self._send_upgrade_ysql_catalog_callhome(success,
                                                               time_taken))
        if ret_code:
            self.shutdown_callhome_thread(
                CALLHOME_EVENT_UPGRADE_YSQL_CATALOG, timeout=5)
            Output.log_error_and_exit("YSQL catalog upgrade failed. Please check the logs.")
        else:
            Output.update_animation("YSQL catalog upgrade successful.")
            self.configs.saved_data["catalog_upgrade_completed"] = True
            self.configs.save_configs()

        status_details = [
            (Output.make_yellow("Status"), "YSQL catalog upgrade successful."),
        ]
        self.shutdown_callhome_thread(
            CALLHOME_EVENT_UPGRADE_YSQL_CATALOG, timeout=5)
        Output.print_out(self.get_status_string_common(status_details))

    def read_pidfile(self, pidfile_path):
        try:
            with open(pidfile_path, 'r') as pidfile:
                pid = int(pidfile.read().strip())
                return pid
        except FileNotFoundError:
            Output.log(f"PID file {pidfile_path} not found.", logging.DEBUG)
        except ValueError:
            Output.log(f"PID file {pidfile_path} contains invalid PID.", logging.DEBUG)
        except Exception as e:
            Output.log(f"An error occurred while reading the PID file: {e}", logging.DEBUG)
        return None

    # Check that `openssl` binary is in our PATH
    # This actually returns true when it is _not_ present!
    def check_openssl(self):
        return shutil.which('openssl') == None

    # Transparent Hugepage check.  Not all distros have this enabled so we need to check for
    # /sys/kernel/mm/transparent_hugepage before checking enabled
    def linux_transparent_hugepage_check(self):
        thp_dir = "/sys/kernel/mm/transparent_hugepage"
        thp_mode = "disabled"
        if os.path.exists(thp_dir):
            with open(os.path.join(thp_dir, 'enabled'), 'r') as thp_enabled:
                thp_enabled = thp_enabled.read()
                thp_mode = re.search(r"\[(.*)\]", thp_enabled).group(1)
        return thp_mode == 'always'

    def check_ports(self):
        failed_ports = []
        warning_ports = []
        mandatory_port_available = True
        recommended_port_available = True
        advertise_ip = self.advertise_ip()
        mandatory_ports = self.get_mandatory_ports()
        recommended_ports = self.get_recommended_ports()

        for port in mandatory_ports:
            if not self.is_port_available(advertise_ip,
                                          self.configs.saved_data.get(port)):
                mandatory_port_available = False
                failed_ports.append(str(self.configs.saved_data.get(port)))

        for port in recommended_ports:
            if not self.is_port_available(advertise_ip,
                                          self.configs.saved_data.get(port)):
                recommended_port_available = False
                warning_ports.append(port)
                PREREQS_ERROR_MSGS[port] = PREREQS_ERROR_MSGS[port].format(
                    self.configs.saved_data.get(port)
                )

        return (failed_ports, warning_ports,
                mandatory_port_available, recommended_port_available)

    def check_python_version_for_rag(self):
        """Check if Python 3.11+ is available for RAG service.

        Returns:
            tuple: (python_exe, is_valid) where python_exe is the path and is_valid is boolean
        """
        for python_executable in ['python3.11', 'python3.12', 'python3.13', 'python3', 'python']:
            try:
                result = subprocess.check_output([python_executable, '--version'],
                                                stderr=subprocess.STDOUT,
                                                universal_newlines=True)
                version_str = result.split()[1]  # "Python 3.11.9" -> "3.11.9"
                major_version, minor_version = map(int, version_str.split('.')[:2])
                if major_version > 3 or (major_version == 3 and minor_version >= 11):
                    Output.log("Found {} for RAG agent virtual environment".format(result.strip()))
                    return (python_executable, True)
            except (subprocess.CalledProcessError, FileNotFoundError, ValueError, IndexError):
                continue
        return (None, False)

    # All Pre-requisites check for Linux
    def linux_prereqs_check(self):
        prereqs_failed = set()
        prereqs_warn = set()
        prereqs_failed_flag = False
        prereqs_warn_flag = False

        if os.path.exists("/.dockerenv"):
            Output.log("Running in a Docker container. Skipping pre-requisite checks.")
            return (prereqs_failed_flag, prereqs_failed, prereqs_warn_flag, prereqs_warn,
                    True, True)

        # Check Python 3.11+ if RAG service is enabled
        tserver_flags = self.configs.saved_data.get("tserver_flags", "")
        if "enable_pg_dist_rag_service=true" in tserver_flags:
            python_exe, is_valid = self.check_python_version_for_rag()
            if not is_valid:
                prereqs_failed.add('python_version')
                prereqs_failed_flag = True

        if not self.linux_transparent_hugepage_check():
            prereqs_warn.add('transparent_hugepages')
            prereqs_warn_flag = True

        ntp_check = shutil.which('ntpd')
        chrony_check = shutil.which('chronyd')
        if not ntp_check and not chrony_check:
            prereqs_warn.add('ntp/chrony')
            prereqs_warn_flag = True

        # TODO: Uncomment this block when clockbound becomes GA.
        # Configuring clockbound is strongly recommended for AWS clusters.
        if not self.assert_system_configured_for_clockbound():
            if self.configs.temp_data["enhance_time_sync_via_clockbound"]:
                prereqs_failed.add('clockbound_fail')
                prereqs_failed_flag = True
            elif chrony_check and using_time_sync_service():
                prereqs_warn.add('clockbound')
                prereqs_warn_flag = True
        else:
            self.configs.temp_data["is_clockbound_configured"] = True

        (failed_ports, warning_ports, mandatory_port_available,
        recommended_port_available) = self.check_ports()

        if not mandatory_port_available:
            prereqs_failed.add('mandatory_ports')
            PREREQS_ERROR_MSGS['mandatory_ports'] = PREREQS_ERROR_MSGS['mandatory_ports'].format(
                    ', '.join(failed_ports), Output.make_yellow("yugabyted start")
                )

        if not recommended_port_available:
            prereqs_warn.update(warning_ports)

        return (prereqs_failed_flag, prereqs_failed, prereqs_warn_flag, prereqs_warn,
                mandatory_port_available, recommended_port_available)

    # All Pre-requistes check for MacOS
    def mac_prereqs_check(self):
        prereqs_failed = set()
        prereqs_warn = set()
        prereqs_failed_flag = False
        prereqs_warn_flag = False

        # Check Python 3.11+ if RAG service is enabled
        tserver_flags = self.configs.saved_data.get("tserver_flags", "")
        if "enable_pg_dist_rag_service=true" in tserver_flags:
            python_exe, is_valid = self.check_python_version_for_rag()
            if not is_valid:
                prereqs_failed.add('python_version')
                prereqs_failed_flag = True

        (failed_ports, warning_ports, mandatory_port_available,
        recommended_port_available) = self.check_ports()

        if not mandatory_port_available:
            prereqs_failed.add('mandatory_ports')
            PREREQS_ERROR_MSGS['mandatory_ports'] = PREREQS_ERROR_MSGS['mandatory_ports'].format(
                    ', '.join(failed_ports), Output.make_yellow("yugabyted start")
                )

        if not recommended_port_available:
            prereqs_warn.update(warning_ports)

        return (prereqs_failed_flag, prereqs_failed, prereqs_warn_flag, prereqs_warn,
                mandatory_port_available, recommended_port_available)

    def is_port_available(self, advertise_ip, port):
        if is_ipv6(advertise_ip):
            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
        else:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            return s.connect_ex((advertise_ip, int(port))) != 0
        finally:
            s.close()

    def get_mandatory_ports(self):
        mandatory_ports = []
        if not self.configs.saved_data.get("read_replica"):
            mandatory_ports.append("master_rpc_port")
            mandatory_ports.append("master_webserver_port")
        mandatory_ports.append("tserver_rpc_port")
        mandatory_ports.append("tserver_webserver_port")
        mandatory_ports.append("ysql_port")
        mandatory_ports.append("ycql_port")

        return mandatory_ports

    def get_recommended_ports(self):
        recommended_ports = []
        if not self.configs.saved_data.get("read_replica") and self.configs.saved_data.get("ui"):
            recommended_ports.append("yugabyted_ui_port")
        recommended_ports.append("ysql_metric_port")
        recommended_ports.append("ycql_metric_port")

        return recommended_ports

    # Setup environment for Distributed RAG service
    def setup_dist_rag_python_env(self, is_first_run):
        """Setup Python virtual environment for the Distributed RAG service.

        This creates a venv and installs requirements for the RAG agent
        located in python/ai/rag_agent.
        - On first run: Always creates a fresh venv
        - On restart: Only creates if venv is missing, otherwise reuses existing

        Args:
            is_first_run: True if this is the first startup, False for restarts

        Returns:
            dict: Result dictionary with 'msg' and 'status' keys similar to prereqs_check
        """
        # The RAG agent is located at python/ai/rag_agent/
        # This path is consistent across development builds and release tarballs
        script_dir = os.path.join(YUGABYTE_DIR, "python", "ai", "rag_agent")
        script_path = os.path.join(script_dir, "start_rag_agent.py")

        if not os.path.exists(script_dir):
            Output.log_error_and_exit(
                "RAG agent directory not found: {}".format(script_dir))

        if not os.path.isfile(script_path):
            Output.log_error_and_exit(
                "RAG agent script not found: {}".format(script_path))

        venv_dir = os.path.join(script_dir, 'venv')
        requirements_path = os.path.join(script_dir, 'requirements.txt')

        # Get Python 3.11+ (prereqs_check already validated this exists)
        python_exe, is_valid = self.check_python_version_for_rag()
        if not is_valid:
            # This should not happen as prereqs_check should have caught it
            Output.log_error_and_exit(
                "Python 3.11+ is required for the RAG agent. "
                "Please install Python 3.11 or higher.")

        # Determine if we need to create the venv
        should_create_venv = False
        if is_first_run:
            # On first run, always create fresh venv
            should_create_venv = True
            if os.path.exists(venv_dir):
                Output.log("Removing existing virtual environment at {}...".format(venv_dir))
                import shutil
                shutil.rmtree(venv_dir)
        else:
            # On restart, only create if venv is missing
            if not os.path.exists(venv_dir):
                should_create_venv = True
            else:
                Output.log("Using existing RAG virtual environment at {}".format(venv_dir))

        # Create virtual environment if needed
        if should_create_venv:
            # Stage 1: Setting up environment
            Output.init_animation("Setting up Distributed RAG environment...")

            # Step 1: Create virtual environment
            Output.log("Creating Python virtual environment at {}...".format(venv_dir))
            try:
                subprocess.check_call([python_exe, '-m', 'venv', venv_dir])
                Output.log("Virtual environment created successfully")
            except subprocess.CalledProcessError as e:
                Output.update_animation(
                    "Failed to set up Distributed RAG environment", Output.ANIMATION_FAIL)
                Output.log_error_and_exit(
                    "Failed to create virtual environment: {}".format(e))

            # Step 2: Install/upgrade pip
            pip_exe = os.path.join(venv_dir, 'bin', 'pip')
            Output.log("Upgrading pip...")
            try:
                subprocess.check_call(
                    [pip_exe, 'install', '--upgrade', 'pip'],
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.DEVNULL)
                Output.log("Pip upgraded successfully")
            except subprocess.CalledProcessError:
                Output.log("Warning: Failed to upgrade pip, continuing anyway...")

            # Stage 2: Installing dependencies (update the animation message)
            Output.update_animation(
                "Installing RAG dependencies (this may take a few minutes)...",
                Output.ANIMATION_RUNNING)

            # Step 3: Install requirements
            if os.path.exists(requirements_path):
                Output.log("Installing Python dependencies from {}...".format(requirements_path))
                try:
                    result = subprocess.run(
                        [pip_exe, 'install', '-r', requirements_path],
                        capture_output=True,
                        text=True,
                        check=True)
                    Output.log("Python dependencies installed successfully")
                except subprocess.CalledProcessError as e:
                    Output.update_animation(
                        "Failed to install Distributed RAG dependencies", Output.ANIMATION_FAIL)
                    error_msg = "Failed to install requirements: {}".format(e)
                    if e.stderr:
                        # Include last 2000 chars of stderr
                        error_msg += "\n\nPip error output:\n{}".format(
                            e.stderr[-2000:])
                    Output.log_error_and_exit(error_msg)
            else:
                Output.log("Warning: No requirements.txt found at {}".format(requirements_path))
                Output.print_and_log("No requirements.txt found at {}".format(requirements_path))

            # Stage 3: Complete (update animation with success message)
            Output.update_animation("Distributed RAG environment ready", Output.ANIMATION_SUCCESS)

        # Return success result like prereqs_check
        return {
            'msg': Output.ANIMATION_SUCCESS + " " + "Distributed RAG setup",
            'status': Output.ANIMATION_SUCCESS
        }

    # Checks whether the prerequisites are met or not
    def prereqs_check(self, ulimits=False):
        help_links = []
        # Check pre-reqs as per machine OS
        if OS_NAME == "Linux":
            check = self.linux_prereqs_check()
        else:
            check = self.mac_prereqs_check()
        # Get pre-req failures and warnings
        prereqs_failed_flag, prereqs_failed, prereqs_warn_flag, prereqs_warn, \
        mandatory_port_available, recommended_port_available = check
        # if prereqs_warn_flag:
        if OS_NAME == "Linux":
            help_links.append("- Quick start for Linux: " +
                                Output.make_underline(QUICK_START_LINKS['linux']))
        else:
            help_links.append("- Quick start for macOS: " +
                                Output.make_underline(QUICK_START_LINKS['mac']))

        if not mandatory_port_available or not recommended_port_available:
            help_links.append("- Default ports: " + Output.make_underline(DEFAULT_PORTS_LINK))

        help_msg = "Please review the following docs and rerun the start command:\n" + \
        '\n'.join(help_links)
        msg = " System checks. Following pre-reqs are not met:\n"

        # Display message for failures
        failed_msg = ""
        for prereq in prereqs_failed:
            failed_msg+=("- "+PREREQS_ERROR_MSGS[prereq]+"\n")

        # Display message for warnings
        warning_msg = ""
        warnings = dict()
        for prereq in prereqs_warn:
            warning_msg+=("- "+PREREQS_ERROR_MSGS[prereq]+"\n")
            warnings[str(prereq)] = str(PREREQS_ERROR_MSGS[prereq])

        # Ulimits display message
        ulimit_msg = {
            'open_files' :"- "+PREREQS_ERROR_MSGS['open_files']+"\n",
            'max_user_processes' :"- "+PREREQS_ERROR_MSGS['max_user_processes']+"\n",
        }

        final_msg = ""
        check_status = None
        # Final display message in case of failures
        if prereqs_failed_flag or not mandatory_port_available:
            final_msg += (Output.ANIMATION_FAIL + " " + Output.make_red("FAILED") + ": " + msg)
            final_msg += failed_msg
            if warning_msg:
                final_msg += warning_msg
            if ulimits:
                for ulimit in ulimits:
                    final_msg+=ulimit_msg[ulimit]
            final_msg += ("\n"+help_msg+"\n")
            check_status = Output.ANIMATION_FAIL
        # Final display message in case of warnings
        elif prereqs_warn_flag or not recommended_port_available:
            final_msg = warnings
            if ulimits:
                for ulimit in ulimits:
                    final_msg[str(ulimit)] = str(PREREQS_ERROR_MSGS[ulimit])
            final_msg["help_msg"] = help_msg
            check_status = Output.ANIMATION_WARNING
        # Final display message in case of Ulimits failures
        elif ulimits:
            final_msg = dict()
            for ulimit in ulimits:
                final_msg[str(ulimit)] = str(PREREQS_ERROR_MSGS[ulimit])
            final_msg["help_msg"] = help_msg
            check_status = Output.ANIMATION_WARNING
        # All pre-reqs passsed
        else:
            final_msg = Output.ANIMATION_SUCCESS + " " + "System checks"
            check_status = Output.ANIMATION_SUCCESS

        result = {
            'msg' : final_msg,
            'status' : check_status,
        }

        return result

    # Checks yb-master and yb-tserver are running. Returns failed processes.
    # TODO: Check postmaster.pid.
    def get_failed_node_processes(self):
        failed_processes = []
        for process in ["tserver",]:
            if not ProcessManager.is_process_running(
                    process, self.configs.saved_data.get("data_dir")):
                failed_processes.append("yb-{}".format(process))

        if not self.configs.saved_data.get("read_replica"):
            for process in ["master",]:
                if not ProcessManager.is_process_running(
                        process, self.configs.saved_data.get("data_dir")):
                    failed_processes.append("yb-{}".format(process))

        return failed_processes

    # Called after receiving certain signals or on exit. Kills all subprocesses.
    def kill_children(self, signum=None, frame=None):
        if signum:
            Output.log("Received signal: {}".format(signum), logging.DEBUG)
        Output.print_and_log("Shutting down...")
        Output.console_access = False
        self.script.daemon_success.put(-1)

        # Shutdown all callhome threads gracefully
        self.shutdown_callhome_thread(timeout=5)

        cur_pid = os.getpid()
        pgid = os.getpgid(cur_pid)
        if not pgid:
            Output.log(
                "PGID could not be found for PID {}. Is {} running?".format(cur_pid, SCRIPT_NAME))
            os._exit(os.EX_OK)
        self.set_signals(SIG_DFL)

        for p in self.processes.values():
            p.delete_pidfile()
        self.script.delete_pidfile()

        try:
            # Kill process group instead of self.processes to ensure
            # any spawned child processes are killed. Use SIGKILL because YugaWare
            # requires KILL signal to terminate and nodes currently do not gracefully terminate.
            os.killpg(pgid, SIGKILL)
            Output.log(
                "{} may not have terminated properly... "
                "Please check PGID {}.".format(SCRIPT_NAME, pgid))
        except OSError as err:
            Output.log(
                "Failed to kill PGID {}... Is {} running?\n{}".format(pgid, SCRIPT_NAME, str(err)))

        # exit no matter what
        os._exit(os.EX_OK)

    def start_ybc_server(self):

        ybc_server_cmd = self.get_ybc_cmd(self.configs.saved_data.get("data_dir"))
        self.processes["ybc-server"] = YBProcessManager(
                "ybc-server", ybc_server_cmd, self.configs.saved_data.get("log_dir"),
                        self.configs.saved_data.get("data_dir"))
        self.processes.get("ybc-server").start()

    def start_first_master_tserver(self):
        self.processes.get("master").start()

        was_already_setup = self.configs.saved_data.get("cluster_member", False)

        if was_already_setup:
            Output.log("Node was a member of some cluster before. "
                    "Skipping master setup")
        elif not self.setup_master():
            # TODO(sanketh): Make these all throw exceptions instead of excns + return values
            return "Failed to start master {}".format(SCRIPT_NAME)

        self.update_tserver_master_addrs()
        self.processes.get("tserver").start()

        if was_already_setup:
            Output.log("Node was a member of some cluster before. "
                    "Skipping tserver setup")
        elif not self.wait_tserver(cluster_type="primary"):
            return "Failed to start tserver {}".format(SCRIPT_NAME)

        if not was_already_setup:
            self.configs.saved_data["cluster_member"] = True
            master_addresses = self.configs.saved_data.get("current_masters")
            universe_uuid = YBAdminProxy.get_cluster_uuid(master_addresses)
            if universe_uuid and universe_uuid != self.configs.saved_data["universe_uuid"]:
                self.configs.saved_data["universe_uuid"] = universe_uuid
                self.configs.save_configs()

        return None

    def start_first_read_replica_tserver(self):
        was_already_setup = self.configs.saved_data.get("cluster_member", False)

        self.processes.get("tserver").start()
        if was_already_setup:
            Output.log("Node was a member of some cluster before. "
                    "Skipping tserver setup")
        elif not self.wait_tserver(cluster_type="read_replica"):
            return "Failed to start tserver {}".format(SCRIPT_NAME)

        if not was_already_setup:
            self.configs.saved_data["cluster_member"] = True
            master_addresses = self.configs.saved_data.get("current_masters")
            universe_uuid = YBAdminProxy.get_cluster_uuid(master_addresses)
            if universe_uuid and universe_uuid != self.configs.saved_data["universe_uuid"]:
                self.configs.saved_data["universe_uuid"] = universe_uuid
                self.configs.save_configs()

        return None

    # Starts yb-master, yb-tserver, and yugaware processes.
    # After initializing, creates a callhome thread.
    def start_processes(self):

        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        join_ip = self.configs.saved_data.get("join")
        master_addresses  = "{}:{}".format(get_url_from_ip(advertise_ip),
                                           master_rpc_port)
        if join_ip:
            master_addresses  = "{}:{},{}".format(get_url_from_ip(join_ip),
                                                  master_rpc_port, master_addresses)
        was_already_setup = self.configs.saved_data.get("cluster_member", False)

        warnings = []
        warnings_for_ui = []
        warning_help_msg=""
        is_first_run = True

        # Do the pre-req check before forming master and tserver commands
        if is_first_run:
            ulimits_failed = self.script.set_rlimits(print_info=True)
            if ulimits_failed:
                msg = "Failed to meet recommended settings. Ulimits too low - {}.\n".format(
                ", ".join(ulimits_failed))
                ulimit_warn_msg = msg + "Note {} will still run, although it may fail for " \
                    "larger workloads. For more info, see {}".format(SCRIPT_NAME, CONFIG_LINK)
                self.alerts.append((ALERT_WARNING, ULIMIT_ERR_CODE, ulimit_warn_msg))

            prereqs_check_result = self.prereqs_check(ulimits=ulimits_failed)

            if prereqs_check_result['status']==Output.ANIMATION_SUCCESS:
                Output.print_out(prereqs_check_result['msg'])
            elif prereqs_check_result['status']==Output.ANIMATION_WARNING:

                warnings.extend(list(prereqs_check_result['msg'].values())[:-1])
                warning_help_msg = prereqs_check_result['msg']["help_msg"]

                prereqs_check_result['msg'].pop("help_msg")
                warnings_for_ui = []
                for k in prereqs_check_result['msg'].keys():
                    warnings_for_ui.extend([k])
            elif prereqs_check_result['status']==Output.ANIMATION_FAIL:
                Output.print_and_log(prereqs_check_result['msg'])
                sys.exit(1)

        # Setup Distributed RAG environment before starting servers (only if enabled)
        tserver_flags = self.configs.saved_data.get("tserver_flags", "")
        if "enable_pg_dist_rag_service=true" in tserver_flags:
            self.setup_dist_rag_python_env(is_first_run)

        common_gflags = self.get_common_flags()

        yb_master_cmd = self.get_master_cmd(common_gflags)

        upgrade_in_progress = self.configs.saved_data.get("upgrade_in_progress")
        Output.log("Value of saved_data - upgrade_in_progress. {}".format(
            upgrade_in_progress), logging.DEBUG)

        catalog_upgrade_completed = self.configs.saved_data.get("catalog_upgrade_completed")
        Output.log("Value of saved_data - catalog_upgrade_completed. {}".format(
            catalog_upgrade_completed), logging.DEBUG)

        if upgrade_in_progress == "true" and not catalog_upgrade_completed:
            # get the previously know current master list before running the yb-admin command
            previously_known_masters = self.configs.saved_data.get("current_masters")
            Output.init_animation("Verifying the current upgrade status...")
            out, ret_code = YBAdminProxy.check_for_ysql_catalog_upgrade_state(
                previously_known_masters, timeout=30)
            if not ret_code:
                catalog_upgrade_completed = True
                self.configs.saved_data["catalog_upgrade_completed"] = True
                self.configs.save_configs()
                Output.log("catalog_upgrade_completed status is set to True.")

            Output.update_animation("Upgrade status successfully verified")

        if upgrade_in_progress == "true" and not catalog_upgrade_completed:
            yb_tserver_cmd = self.configs.saved_data.get("old_release_cmd")
            yb_tserver_cmd.extend(["--ysql_yb_major_version_upgrade_compatibility=11"])
            Output.log("Catalog upgrade is in progress. " +
                       "Starting the tserver with old release path.")
        else:
            yb_tserver_cmd = self.get_tserver_cmd(common_gflags)

        self.processes = {
            "master": YBProcessManager(
                "master", yb_master_cmd, self.configs.saved_data.get("log_dir"),
                self.configs.saved_data.get("data_dir")),
            "tserver": YBProcessManager(
                "tserver", yb_tserver_cmd, self.configs.saved_data.get("log_dir"),
                self.configs.saved_data.get("data_dir")),
        }

        for p in self.processes.values():
            pid = p.get_pid()
            if pid:
                Output.print_out(
                    "{} is already running... Is there an existing {} process?".format(
                        p.name, SCRIPT_NAME))
                # Clear self.processes so kill_children() doesn't kill existing processes.
                self.processes = {}
                return

        masters_list_update_thread = None
        #Start the different thread for extracting the YBC binaries
        ybc_extract_thread = None
        while True:

            is_first_install = is_first_run and not self.is_yb_initialized()

            # Create data directory.
            data_dir = self.configs.saved_data.get("data_dir")
            if not os.path.exists(data_dir):
                Output.log(
                    "Creating data directory {}.".format(data_dir))
                os.makedirs(data_dir)

            # Delete corrupted data dirs left from interrupting yb-master and yb-tserver startup.
            pid_file_name = os.path.basename(self.script.pidfile)
            data_dir_files = [ x for x in os.listdir(data_dir) if x != pid_file_name ]
            if is_first_install and data_dir_files and upgrade_in_progress == "false":
                Output.print_and_log(
                    ("Found files {} in data dir {} from possibly failed initialization."
                    " Removing...").format(data_dir_files, data_dir))
                rmcontents(data_dir, exclude_names=[pid_file_name])

            if self.configs.saved_data.get("backup_daemon"):
                if not os.path.exists(os.path.join(YUGABYTE_DIR, "ybc")):
                    ybc_extract_thread = Thread(target=download_extract_ybc_binary)
                    ybc_extract_thread.daemon = True
                    ybc_extract_thread.start()

            # Start or initialize yb-master and yb-tserver.
            if is_first_run:
                Output.init_animation("Starting the YugabyteDB Processes...")

                self.post_install_yb()

                ret = self.start_first_master_tserver()
                if ret:
                    Output.update_animation("Database failed to start",
                        status=Output.ANIMATION_FAIL)
                    Output.log_error_and_exit(ret)

                Output.update_animation("YugabyteDB Started")

                if join_ip:
                    Output.print_and_log(Output.ANIMATION_SUCCESS +
                            " Node joined a running cluster with UUID {}"
                            .format(self.configs.saved_data.get("universe_uuid")))

                if self.configs.saved_data.get("secure"):
                    Output.init_animation("Enabling Encryption in Transit and " +
                        "Password Authentication...")
                    if not join_ip and not was_already_setup:
                        alphadigits = string.ascii_letters + string.digits
                        new_password = ''.join(PASSWORD_GENNERATOR(alphadigits) for i in range(12))
                        self.update_db_passwords(new_password)
                        self.create_password_file()
                    Output.update_animation("Encryption in Transit and " +
                        "Password Authentication enabled")
                else:
                    insecure_msg = "Cluster started in an insecure mode without " + \
                        "authentication and encryption enabled. For non-production use only, " + \
                        "not to be used without firewalls blocking the internet traffic."
                    warnings.append(insecure_msg)
                    warnings_for_ui.extend(["insecure"])

                # reset the upgrade state variables once the catalog upgrade is complete
                upgrade_in_progress = self.configs.saved_data.get("upgrade_in_progress")
                catalog_upgrade_completed = self.configs.saved_data.get("catalog_upgrade_completed")
                if upgrade_in_progress == "true" and catalog_upgrade_completed:
                    self.configs.saved_data["upgrade_in_progress"] = "false"
                    self.configs.saved_data["catalog_upgrade_completed"] = "false"
                    self.configs.saved_data["old_release_cmd"] = []
                    Output.log("YSQL Catalog Upgrade complete: Upgrade in progress is set" +
                                "to false and old_release_cmd is reset.")

                # Persist the config after successful start
                self.configs.save_configs()
            else:
                for name in ("master", "tserver"):
                    process = self.processes.get(name)
                    process.remove_error_logs()
                    if not process.is_running():
                        Output.log(
                            "{} died unexpectedly. Restarting...".format(process.name),
                            logging.ERROR)
                        if name == "tserver":
                            self.update_tserver_master_addrs()
                        process.start()

            if is_first_run:
                ui_port_available = self.configs.temp_data.get("ui_port_available")
                if self.configs.saved_data.get("ui") and ui_port_available:
                    yugabyted_ui_path = find_binary_location("yugabyted-ui")
                    if (yugabyted_ui_path is None):
                        ui_bin_not_found = "Couldn't find yugabyted-ui binary. " + \
                            "Cluster started without UI."
                        warnings.append(ui_bin_not_found)
                        self.configs.saved_data["ui"] = False
                    else:
                        yugabyted_ui_cmd = [ yugabyted_ui_path ] + \
                            [
                                "-database_host={}".format(advertise_ip),
                                "-master_ui_port={}".format(self.configs.saved_data.
                                                            get("master_webserver_port")),
                                "-tserver_ui_port={}".format(self.configs.saved_data.
                                                            get("tserver_webserver_port")),
                                "-data_dir={}".format(self.configs.saved_data.
                                                            get("data_dir")),
                                "-conf_file={}".format(self.conf_file),
                                "-warnings={}".format("|".join(warnings_for_ui))
                            ]
                        if self.configs.saved_data.get("secure"):
                            yugabyted_ui_cmd.extend(["-secure=true",
                                "-ysql_password={}".format(
                                self.configs.saved_data.get("database_password")),
                                "-ycql_password={}".format(
                                self.configs.saved_data.get("database_password"))])

                        if self.configs.saved_data.get("ysql_port"):
                            yugabyted_ui_cmd.append("-ysql_port={}".format(
                                self.configs.saved_data.get("ysql_port")))
                        if self.configs.saved_data.get("ycql_port"):
                            yugabyted_ui_cmd.append("-ycql_port={}".format(
                                self.configs.saved_data.get("ycql_port")))

                        self.processes["yugabyted-ui"] = ProcessManager(
                            "yugabyted-ui", yugabyted_ui_cmd,
                            self.configs.saved_data.get("log_dir"),
                            self.configs.saved_data.get("data_dir"))


            if ybc_extract_thread is not None:
                ybc_extract_thread.join()

            if self.configs.saved_data.get("backup_daemon"):
                if self.processes.get("ybc-server") is None:
                    self.start_ybc_server()

            if self.configs.saved_data.get("ui") and ui_port_available:
                (_, _) = self.verify_start_yugabyted_ui(is_first_run, is_first_install)

            if is_first_install and not join_ip:
                self.first_install_init_auth()
                self.configs.saved_data["created_on"] = int(time.time())
                self.configs.save_configs()

            if warnings:
                warning_msg = "\n" + Output.make_yellow(Output.ANIMATION_WARNING +
                    " WARNINGS") + ":\n"

                for msg in warnings:
                    warning_msg += "- " + msg + "\n"
                if warning_help_msg:
                    warning_msg += "\n" + warning_help_msg

            if is_first_run:
                status, _ = self.get_status_string()
                status += "{} YugabyteDB started successfully! To load a sample dataset, " \
                    "try '{} demo'.\n" \
                    "{} Join us on Slack at {}\n" \
                    "{} Claim your free t-shirt at {}\n".format(
                        Output.ROCKET, SCRIPT_NAME, Output.PARTY,
                        Output.make_underline(SLACK_LINK), Output.SHIRT,
                        Output.make_underline(COMMUNITY_REWARDS_LINK))

                if self.configs.saved_data.get("secure"):
                    if not join_ip:
                        status += "\n{} is saved in the {}.\n{} is stored at {}\n".format(
                            Output.make_cyan("DB_PASSWORD"),
                            Output.make_cyan("Credentials File"),
                            Output.make_cyan("Credentials File"),
                            os.path.join(self.configs.saved_data.get("data_dir"),
                                "{}_credentials.txt".format(SCRIPT_NAME)))
                    else:
                        status += "\n{} is saved in ".format(Output.make_cyan("DB_PASSWORD")) + \
                            "the {} which has been generated on the 1st node.\n".format(
                                Output.make_cyan("Credentials File"))

                if len(self.setup_env_init.get_ysql_password()) > 99:
                    status = status + Output.make_red(YSQL_PASSWORD_LENGTH_WARNING)

                if warnings:
                    Output.print_out(warning_msg)

                Output.print_out(status)

                if self.configs.temp_data.get("background"):
                    # Let original process know daemon was successful so it can exit.
                    # This is to display the initial status message.
                    self.script.daemon_success.put(1)
                    # Ignore any console output as important information will be logged.
                    with open('/dev/null', 'r+') as dev_null:
                        Output.console_access = False
                        sys.stderr.flush()
                        sys.stdout.flush()
                        os.dup2(dev_null.fileno(), sys.stdin.fileno())
                        os.dup2(dev_null.fileno(), sys.stderr.fileno())
                        os.dup2(dev_null.fileno(), sys.stdout.fileno())

                self.diagnostics.first_run_secs = time.time() - start_time_sec
                self.diagnostics.first_install = is_first_install

                if self.configs.saved_data.get("callhome"):
                    self.start_callhome_thread(
                        CALLHOME_EVENT_START,
                        self.start_cmd_callhome,
                        use_shutdown_event=False)

                    self.start_callhome_thread(
                        CALLHOME_EVENT_HEARTBEAT_STATUS,
                        self.heartbeat_status_callhome_loop,
                        use_shutdown_event=True)

                masters_list_update_thread = Thread(target=self.update_masters_list_loop)
                masters_list_update_thread.daemon = True
                masters_list_update_thread.start()

                if join_ip:
                    created_on_timestamp_thread = Thread(target=self.get_created_on_timestamp_loop)
                    created_on_timestamp_thread.daemon = True
                    created_on_timestamp_thread.start()

            if self.configs.saved_data.get("callhome"):
                self.shutdown_callhome_thread(callhome_event_name=CALLHOME_EVENT_START, timeout=120)

            is_first_run = False

            time.sleep(int(self.configs.saved_data.get("polling_interval")))

    def verify_start_yugabyted_ui(self, is_first_run, is_first_install):

        was_started = False
        err = None

        if not self.configs.saved_data.get("ui"):
            return (err, was_started)

        yugabyted_ui_process = self.processes.get("yugabyted-ui")

        try:
            if is_first_run:
                Output.init_animation("Bringing up UI...")

            # Start yugabyted UI process.
            if not yugabyted_ui_process.is_running():
                if not is_first_run:
                    Output.log(
                        "Webserver died unexpectedly. Restarting...", logging.ERROR)
                yugabyted_ui_process.start()
                was_started = True

            # After first run, do not attempt any more setup, just return.
            if not is_first_run:
                return (err, was_started)

            if is_first_run:
                Output.update_animation("UI ready")

        finally:
            if is_first_run:
                animation_status = Output.ANIMATION_FAIL if err else Output.ANIMATION_SUCCESS
                # Output.update_animation("UI status", status=animation_status)
        return (err, was_started)

    def start_rr_process(self):
        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        join_ip = self.configs.saved_data.get("join")
        master_addresses  = "{}:{}".format(get_url_from_ip(advertise_ip),
                                           master_rpc_port)
        if join_ip:
            master_addresses  = "{}:{},{}".format(get_url_from_ip(join_ip),
                                                  master_rpc_port, master_addresses)

        is_first_run = True
        warnings = []
        warning_help_msg=""
        if is_first_run:
            ulimits_failed = self.script.set_rlimits(print_info=True)
            if ulimits_failed:
                msg = "Failed to meet recommended settings. Ulimits too low - {}.\n".format(
                ", ".join(ulimits_failed))
                ulimit_warn_msg = msg + "Note {} will still run, although it may fail for " \
                    "larger workloads. For more info, see {}".format(SCRIPT_NAME, CONFIG_LINK)
                self.alerts.append((ALERT_WARNING, ULIMIT_ERR_CODE, ulimit_warn_msg))

            prereqs_check_result = self.prereqs_check(ulimits=ulimits_failed)
            if prereqs_check_result['status']==Output.ANIMATION_SUCCESS:
                Output.print_out(prereqs_check_result['msg'])
            elif prereqs_check_result['status']==Output.ANIMATION_WARNING:
                warnings.extend(list(prereqs_check_result['msg'].values())[:-1])
                warning_help_msg = prereqs_check_result['msg']["help_msg"]
            elif prereqs_check_result['status']==Output.ANIMATION_FAIL:
                Output.print_and_log(prereqs_check_result['msg'])
                sys.exit(1)


        upgrade_in_progress = self.configs.saved_data.get("upgrade_in_progress")
        Output.log("Value of saved_data - upgrade_in_progress. {}".format(
            upgrade_in_progress), logging.DEBUG)
        catalog_upgrade_completed = self.configs.saved_data.get("catalog_upgrade_completed")
        Output.log("Value of saved_data - catalog_upgrade_completed. {}".format(
            catalog_upgrade_completed), logging.DEBUG)

        common_gflags = self.get_common_flags()

        yb_tserver_cmd = self.get_tserver_cmd(common_gflags)

        if upgrade_in_progress == "true" and not catalog_upgrade_completed:
            # get the previously know current master list before running the yb-admin command
            previously_known_masters = self.configs.saved_data.get("current_masters")
            Output.init_animation("Verifying the current upgrade status...")
            out, ret_code = YBAdminProxy.check_for_ysql_catalog_upgrade_state(
                previously_known_masters, timeout=30)
            if not ret_code:
                catalog_upgrade_completed = True
                self.configs.saved_data["catalog_upgrade_completed"] = True
                self.configs.save_configs()
                Output.log("catalog_upgrade_completed status is set to True.")

            Output.update_animation("Upgrade status successfully verified")

        if upgrade_in_progress == "true" and not catalog_upgrade_completed:
            yb_tserver_cmd = self.configs.saved_data.get("old_release_cmd")
            yb_tserver_cmd.extend(["--ysql_yb_major_version_upgrade_compatibility=11"])
            Output.log("Catalog upgrade is in progress. " +
                       "Starting the tserver with old release path.")

        self.processes = {
            "tserver": YBProcessManager(
                "tserver", yb_tserver_cmd, self.configs.saved_data.get("log_dir"),
                self.configs.saved_data.get("data_dir")),
        }

        for p in self.processes.values():
            pid = p.get_pid()
            if pid:
                Output.print_out(
                    "{} is already running... Is there an existing {} process?".format(
                        p.name, SCRIPT_NAME))
                # Clear self.processes so kill_children() doesn't kill existing processes.
                self.processes = {}
                return

        # callhome_thread = None
        masters_list_update_thread = None
        self.stop_callhome = False
        while True:
            should_callhome = False

            is_first_install = is_first_run and not self.is_yb_initialized()

            # Create data directory.
            data_dir = self.configs.saved_data.get("data_dir")
            if not os.path.exists(data_dir):
                Output.log(
                    "Creating data directory {}.".format(data_dir))
                os.makedirs(data_dir)

            # Delete corrupted data dirs left from interrupting yb-master and yb-tserver startup.
            pid_file_name = os.path.basename(self.script.pidfile)
            data_dir_files = [ x for x in os.listdir(data_dir) if x != pid_file_name ]
            if is_first_install and data_dir_files and upgrade_in_progress == "false":
                Output.print_and_log(
                    ("Found files {} in data dir {} from possibly failed initialization."
                    " Removing...").format(data_dir_files, data_dir))
                rmcontents(data_dir, exclude_names=[pid_file_name])

            # Start or initialize yb-master and yb-tserver.
            if is_first_run:
                Output.init_animation("Starting the YugabyteDB Processes...")

                self.post_install_yb()

                ret = self.start_first_read_replica_tserver()
                if ret:
                    Output.update_animation("Database failed to start",
                        status=Output.ANIMATION_FAIL)
                    Output.log_error_and_exit(ret)

                Output.update_animation("YugabyteDB Started")

                if join_ip:
                    Output.print_and_log(Output.ANIMATION_SUCCESS +
                            " Read Replica node joined a running cluster with UUID {}"
                            .format(self.configs.saved_data.get("universe_uuid")))

                if self.configs.saved_data.get("secure"):
                    Output.init_animation("Enabling Encryption in Transit and " +
                        "Password Authentication...")
                    Output.update_animation("Encryption in Transit and " +
                        "Password Authentication enabled")
                else:
                    warnings.append("Cluster started in an insecure mode without " +
                        "authentication and encryption enabled. For non-production use only, " +
                        "not to be used without firewalls blocking the internet traffic.")


                # reset the upgrade state variables once the catalog upgrade is complete
                upgrade_in_progress = self.configs.saved_data.get("upgrade_in_progress")
                catalog_upgrade_completed = self.configs.saved_data.get("catalog_upgrade_completed")
                if upgrade_in_progress == "true" and catalog_upgrade_completed:
                    self.configs.saved_data["upgrade_in_progress"] = "false"
                    self.configs.saved_data["catalog_upgrade_completed"] = "false"
                    self.configs.saved_data["old_release_cmd"] = []
                    Output.log("YSQL Catalog Upgrade complete: Upgrade in progress is set" +
                                "to false and old_release_cmd is reset.")

                # Persist the config after successful start
                self.configs.save_configs()
            else:
                for name in ("tserver",):
                    process = self.processes.get(name)
                    process.remove_error_logs()
                    if not process.is_running():
                        Output.log(
                            "{} died unexpectedly. Restarting...".format(process.name),
                            logging.ERROR)
                        self.update_tserver_master_addrs()
                        process.start()
                        should_callhome = True

            if warnings:
                warning_msg = "\n" + Output.make_yellow(Output.ANIMATION_WARNING +
                    " WARNINGS") + ":\n"

                for msg in warnings:
                    warning_msg += "- " + msg + "\n"
                if warning_help_msg:
                    warning_msg += "\n" + warning_help_msg

            if is_first_run:
                status, _ = self.get_status_string()
                status += "{} YugabyteDB started successfully! To load a sample dataset, " \
                    "try '{} demo'.\n" \
                    "{} Join us on Slack at {}\n" \
                    "{} Claim your free t-shirt at {}\n".format(
                        Output.ROCKET, SCRIPT_NAME, Output.PARTY,
                        Output.make_underline(SLACK_LINK), Output.SHIRT,
                        Output.make_underline(COMMUNITY_REWARDS_LINK))

                if len(self.setup_env_init.get_ysql_password()) > 99:
                    status = status + Output.make_red(YSQL_PASSWORD_LENGTH_WARNING)

                if warnings:
                    Output.print_out(warning_msg)

                Output.print_out(status)

                if self.configs.temp_data.get("background"):
                    # Let original process know daemon was successful so it can exit.
                    # This is to display the initial status message.
                    self.script.daemon_success.put(1)
                    # Ignore any console output as important information will be logged.
                    with open('/dev/null', 'r+') as dev_null:
                        Output.console_access = False
                        sys.stderr.flush()
                        sys.stdout.flush()
                        os.dup2(dev_null.fileno(), sys.stdin.fileno())
                        os.dup2(dev_null.fileno(), sys.stderr.fileno())
                        os.dup2(dev_null.fileno(), sys.stdout.fileno())

                Diagnostics.first_run_secs = time.time() - start_time_sec
                Diagnostics.first_install = is_first_install

                if self.configs.saved_data.get("callhome"):
                    self.start_callhome_thread(
                        CALLHOME_EVENT_START,
                        self.start_cmd_callhome,
                        use_shutdown_event=False
                    )
                    self.start_callhome_thread(
                        CALLHOME_EVENT_HEARTBEAT_STATUS,
                        self.heartbeat_status_callhome_loop,
                        use_shutdown_event=True
                    )

                masters_list_update_thread = Thread(target=self.update_masters_list_loop)
                masters_list_update_thread.daemon = True
                masters_list_update_thread.start()

            if self.configs.saved_data.get("callhome"):
                self.shutdown_callhome_thread(
                    callhome_event_name=CALLHOME_EVENT_START, timeout=120
                )

            is_first_run = False
            if should_callhome:
                self.callhome()

            time.sleep(int(self.configs.saved_data.get("polling_interval")))

    def get_common_flags(self):
        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        join_ip = self.configs.saved_data.get("join")
        fault_tolerance = self.configs.saved_data.get("fault_tolerance")
        master_addresses  = "{}:{}".format(get_url_from_ip(advertise_ip),
                                           master_rpc_port)
        if join_ip:
            master_addresses  = "{}:{},{}".format(get_url_from_ip(join_ip),
                                                  master_rpc_port, master_addresses)
        certs_dir = self.configs.saved_data.get("certs_dir")

        # Check if connection manager is enabled
        tserver_flags = self.configs.saved_data.get("tserver_flags")

        if tserver_flags.find("enable_ysql_conn_mgr") != -1:
            METRICS_SNAPSHOT_LIST.append("ysql_conn_mgr")

        # Check for multiple data directories
        fs_data_dirs = self.configs.saved_data.get("data_dir")
        additional_data_dirs = self.configs.saved_data.get(
                                        "additional_data_dir")
        if additional_data_dirs:
            fs_data_dirs += "," + additional_data_dirs

        common_gflags = [
            "--stop_on_parent_termination",
            "--undefok=stop_on_parent_termination",
            "--fs_data_dirs={}".format(fs_data_dirs),
            "--webserver_interface={}".format(advertise_ip),
            "--metrics_snapshotter_tserver_metrics_whitelist={}".format(
                ",".join(METRICS_SNAPSHOT_LIST)),
            "--yb_num_shards_per_tserver={}".format(YB_NUM_SHARDS_PER_TSERVER),
            "--ysql_num_shards_per_tserver={}".format(YSQL_NUM_SHARDS_PER_TSERVER),
            "--placement_cloud={}".format(self.configs.saved_data.get("cloud_provider")),
            "--placement_region={}".format(self.configs.saved_data.get("cloud_region")),
            "--placement_zone={}".format(self.configs.saved_data.get("cloud_zone")),
            "--use_memory_defaults_optimized_for_ysql=true",
        ]

        if fault_tolerance == "region":
            common_gflags.append("--leader_failure_max_missed_heartbeat_periods=10")

        if self.configs.saved_data.get("secure"):
            common_gflags.extend(["--certs_dir={}".format(certs_dir),
            "--allow_insecure_connections=false",
            "--use_node_to_node_encryption=true",
            "--use_client_to_server_encryption=true",
            "--certs_for_client_dir={}".format(certs_dir),
            "--certs_for_cdc_dir={}/xcluster".format(certs_dir),])

        return common_gflags

    # find the cmdline for a given pid
    def get_cmdline_from_pid(self, pid):
        try:
            with open(f"/proc/{pid}/cmdline", 'r') as f:
                cmdline = f.read().split('\0')[:-1]
            return cmdline
        except FileNotFoundError:
            return None
        except Exception as e:
            return None

    # Returns value of flag_name in flags.
    # Returns None is none exists.
    def get_flag_value(self, flags, flag_name):
        for flag in flags.split(','):
            if flag.startswith(flag_name):
                return flag.split("=")[1]
        return None

    # Requires that flag_name not already be in flags.
    # Returns flag_name=flag_value appended to flags.
    def append_flag(self, flags, flag_name, flag_value):
        if flags:
            flags += ','
        return flags + f"{flag_name}={flag_value}"

    # Parses a flagfile and returns a dictionary of flag names to values.
    # Ignores comments (lines starting with #) and empty lines.
    # Returns empty dict on error.
    def parse_flagfile(self, flagfile_path):
        flag_dict = {}
        try:
            with open(flagfile_path, 'r') as f:
                for line in f:
                    line = line.strip()
                    # Skip comments and empty lines
                    if not line or line.startswith('#'):
                        continue
                    # Parse flag=value format
                    if '=' in line:
                        # Remove leading -- if present
                        if line.startswith('--'):
                            line = line[2:]
                        parts = line.split('=', 1)
                        if len(parts) == 2:
                            flag_name = parts[0].strip()
                            flag_value = parts[1].strip()
                            flag_dict[flag_name] = flag_value
        except (IOError, OSError) as e:
            Output.log("Warning: Failed to read flagfile {}: {}".format(flagfile_path, str(e)))
        return flag_dict

    def get_flagfile_dict(self, flags_str):
        flagfile_value = self.get_flag_value(flags_str, "flagfile")
        if not flagfile_value:
            return {}
        return self.parse_flagfile(flagfile_value)

    # Extracts a simple flag value from flags_str or flagfile_dict.
    # Direct flags in flags_str take precedence over flagfile_dict.
    # Returns the flag value or None if not found.
    def get_user_flag_value(self, flags_str, flagfile_dict, flag_name):
        # First check direct flags (they take precedence)
        if flags_str:
            flags_list = flags_str.split(',')
            for flag in flags_list:
                flag = flag.strip()
                if '=' in flag:
                    key, value = flag.split('=', 1)
                    if key.strip() == flag_name:
                        return value.strip()

        # If not in direct flags, check flagfile
        if flagfile_dict and flag_name in flagfile_dict:
            return flagfile_dict[flag_name]

        return None

    # Extracts a CSV flag value from flags_str or flagfile_dict.
    # Direct flags in flags_str take precedence over flagfile_dict.
    # For direct flags, CSV format is: csv_flag_name={key1=val1,key2=val2}
    # For flagfile, CSV format is: csv_flag_name=key1=val1,key2=val2
    # Returns the csv_key value or None if not found.
    def get_user_csv_flag_value(self, flags_str, flagfile_dict, csv_flag_name, csv_key):
        # First check direct flags (they take precedence)
        # In direct flags, CSV flags are in format: flag_name={key1=val1,key2=val2}
        if flags_str:
            # Use regex to extract content between curly braces
            pattern = r'{}=\{{([^}}]*)\}}'.format(re.escape(csv_flag_name))
            match = re.search(pattern, flags_str)
            if match:
                csv_content = match.group(1)
                # Parse CSV content
                for item in csv_content.split(','):
                    item = item.strip()
                    if '=' in item:
                        key, value = item.split('=', 1)
                        if key.strip() == csv_key:
                            return value.strip()

        # If not in direct flags, check flagfile
        # In flagfile, CSV flags are in format: flag_name=key1=val1,key2=val2
        if flagfile_dict and csv_flag_name in flagfile_dict:
            csv_content = flagfile_dict[csv_flag_name]
            for item in csv_content.split(','):
                item = item.strip()
                if '=' in item:
                    key, value = item.split('=', 1)
                    if key.strip() == csv_key:
                        return value.strip()

        return None

    def config_time_source_clockbound(self, flags):
        # Configure tserver flag time_source=clockbound
        # when clockbound is installed and configured.
        if self.configs.temp_data["is_clockbound_configured"]:
            # Check database configuration.
            time_source = self.get_flag_value(flags, "time_source")
            if time_source and time_source != "clockbound":
                if self.configs.temp_data["enhance_time_sync_via_clockbound"]:
                    raise ValueError("--time_source gflag is already set to {}.".format(
                        time_source) + "Cannot configure time_source with" +
                        " --enhance_time_sync_via_clockbound.")
                else:
                    Output.log("--time_source gflag is already set to {}.".format(time_source) +
                        " Cannot configure time_source to clockbound.")

            # Configure time_source=clockbound if not already.
            if not time_source:
                flags = self.append_flag(flags, "time_source", "clockbound")

            # 100us clock error is a good estimate for PTP configurations.
            if is_phc_configured():
                clock_error_estimate = self.get_flag_value(
                    flags, "clockbound_clock_error_estimate_usec")
                if not clock_error_estimate:
                    flags = self.append_flag(flags,
                        "clockbound_clock_error_estimate_usec", 100)

        return flags

    def get_master_cmd(self, common_flags):
        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        join_ip = self.configs.saved_data.get("join")
        advertise_url_ip = get_url_from_ip(advertise_ip)
        master_addresses  = "{}:{}".format(advertise_url_ip, master_rpc_port)
        if join_ip:
            master_addresses  = "{}:{},{}".format(get_url_from_ip(join_ip),
                                                  master_rpc_port, master_addresses)


        flag_list = common_flags + [
            "--rpc_bind_addresses={}:{}".format(advertise_url_ip, master_rpc_port),
            "--server_broadcast_addresses={}:{}".format(advertise_url_ip, master_rpc_port),
            "--replication_factor=1",
            "--server_dump_info_path={}".format(
                os.path.join(self.configs.saved_data.get("data_dir"), "master-info")),
            "--master_enable_metrics_snapshotter=true",
            "--webserver_port={}".format(self.configs.saved_data.get("master_webserver_port")),
            "--instance_uuid_override={}".format(self.configs.saved_data.get("master_uuid")),
            "--enforce_tablet_replica_limits=true",
            "--split_respects_tablet_replica_limits=true",
        ]

        yb_master_cmd = [find_binary_location("yb-master")]

        master_flags = self.configs.saved_data.get("master_flags","")

        # Parse flagfile if specified
        flagfile_dict = self.get_flagfile_dict(master_flags)

        # Add yb_enable_read_committed_isolation flag
        # Check if user has provided this flag (direct flags or flagfile)
        user_read_committed_value = self.get_user_flag_value(
            master_flags, flagfile_dict, "yb_enable_read_committed_isolation")

        if user_read_committed_value is None:
            # User has not provided the flag, add default value
            if master_flags and not master_flags.endswith(","):
                master_flags += ","
            master_flags += "yb_enable_read_committed_isolation=true"
        # If user provided it, it's already in master_flags or will be read from flagfile
        # No action needed as the existing flag processing will handle it

        master_flags = self.config_time_source_clockbound(master_flags)
        self.configs.saved_data["master_flags"] = master_flags

        if master_flags:
            # This will split the simple and complex flags
            # Now we can parse and append the master flags
            master_flags_list = re.split(r',\s*(?![^{}]*\})',
                                                master_flags)

            for master_flag in master_flags_list:
                # If it's just empty braces, skip appending to the command list
                if re.search(r'=\{\}$', master_flag):
                    continue
                # Check for complex flags and remove curly brackets
                elif re.search(r'={.*}$', master_flag):
                    complex_flag = re.sub(r'({|})', '', master_flag)
                    complex_flag_name = complex_flag.split('=')[0]
                    is_complex_flag_new = True
                    for i in range(len(flag_list)):
                        master_flag = flag_list[i]
                        flag_name = master_flag.split('=', 1)[0]
                        if complex_flag_name == flag_name.lstrip('-'):
                            flag_list[i] += "," + complex_flag.split('=')[1]
                            is_complex_flag_new = False
                            break

                    if is_complex_flag_new:
                        yb_master_cmd.append("--{}".format(complex_flag))
                else:
                    # Handle simple flags
                    yb_master_cmd.append("--{}".format(master_flag))

        yb_master_cmd.extend(flag_list)

        # if a join ip is specified, bring up a shell mode master
        if not join_ip:
            yb_master_cmd.append("--master_addresses={}".format(master_addresses))
            yb_master_cmd.append("--cluster_uuid={}".format(
                self.configs.saved_data.get("universe_uuid")))

        return yb_master_cmd

    def get_tserver_cmd(self, common_flags):
        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        join_ip = self.configs.saved_data.get("join")
        advertise_url_ip = get_url_from_ip(advertise_ip)
        join_url_ip = get_url_from_ip(join_ip)
        master_addresses  = "{}:{}".format(advertise_url_ip, master_rpc_port)
        if join_ip:
            master_addresses  = "{}:{},{}".format(join_url_ip, master_rpc_port,
                                                  master_addresses)
        tserver_rpc_port = self.configs.saved_data.get("tserver_rpc_port")

        yb_tserver_cmd = [find_binary_location("yb-tserver")] + common_flags + \
            [
                "--{}={}".format(TS_MASTER_ADDRS_FLAG, master_addresses),
                "--rpc_bind_addresses={}:{}".format(advertise_url_ip, tserver_rpc_port),
                "--server_broadcast_addresses={}:{}".format(advertise_url_ip, tserver_rpc_port),
                "--cql_proxy_bind_address={}:{}".format(advertise_url_ip,
                                                        self.configs.saved_data.get("ycql_port")),
                "--server_dump_info_path={}".format(
                    os.path.join(self.configs.saved_data.get("data_dir"), "tserver-info")),
                "--start_pgsql_proxy", "--pgsql_proxy_bind_address={}:{}".format(
                    advertise_url_ip, self.configs.saved_data.get("ysql_port")),
                "--tserver_enable_metrics_snapshotter=true",
                "--metrics_snapshotter_interval_ms=11000",
                "--webserver_port={}".format(self.configs.saved_data.get("tserver_webserver_port")),
                "--instance_uuid_override={}".format(self.configs.saved_data.get("tserver_uuid")),
                "--start_redis_proxy=false",
                "--placement_uuid={}".format(self.configs.saved_data.get("placement_uuid")),
            ]

        tserver_flags = self.configs.saved_data.get("tserver_flags","")

        # Parse flagfile if specified
        flagfile_dict = self.get_flagfile_dict(tserver_flags)

        # Add yb_enable_read_committed_isolation flag
        # Check if user has provided this flag (direct flags or flagfile)
        user_read_committed_value = self.get_user_flag_value(
            tserver_flags, flagfile_dict, "yb_enable_read_committed_isolation")

        if user_read_committed_value is None:
            # User has not provided the flag, add default value
            if tserver_flags and not tserver_flags.endswith(","):
                tserver_flags += ","
            tserver_flags += "yb_enable_read_committed_isolation=true"
        # If user provided it, it's already in tserver_flags or will be read from flagfile
        # No action needed as the existing flag processing will handle it

        # Add ysql_pg_conf_csv with yb_enable_cbo
        # Check if user has provided yb_enable_cbo within ysql_pg_conf_csv
        user_cbo_value = self.get_user_csv_flag_value(
            tserver_flags, flagfile_dict, "ysql_pg_conf_csv", "yb_enable_cbo")

        if user_cbo_value is None:
            # User has not provided yb_enable_cbo, need to add it
            # Use regex to check if ysql_pg_conf_csv already exists in tserver_flags
            pattern = r'ysql_pg_conf_csv=\{([^}]*)\}'
            match = re.search(pattern, tserver_flags)

            if match:
                # ysql_pg_conf_csv exists, need to add yb_enable_cbo to it
                existing_csv = match.group(1)
                updated_csv = existing_csv + ",yb_enable_cbo=on" if existing_csv \
                    else "yb_enable_cbo=on"
                tserver_flags = re.sub(pattern,
                                      'ysql_pg_conf_csv={' + updated_csv + '}',
                                      tserver_flags)
            else:
                # ysql_pg_conf_csv doesn't exist, add it
                if tserver_flags and not tserver_flags.endswith(","):
                    tserver_flags += ","
                tserver_flags += "ysql_pg_conf_csv={yb_enable_cbo=on}"
        # If user provided it, it's already in tserver_flags or will be read from flagfile
        # No action needed as the existing flag processing will handle it

        tserver_flags = self.config_time_source_clockbound(tserver_flags)
        self.configs.saved_data["tserver_flags"] = tserver_flags

        if tserver_flags:
            # This will split the simple and complex flags
            # Now we can parse and append the tserver flags
            tserver_flags_list = re.split(r',\s*(?![^{}]*\})',
                                                tserver_flags)

            for tserver_flag in tserver_flags_list:
                # If it's just empty braces, skip appending to the command list
                if re.search(r'=\{\}$', tserver_flag):
                    continue
                # Check for complex flags and remove curly brackets
                elif re.search(r'={.*}$', tserver_flag):
                    complex_flag = re.sub(r'({|})', '', tserver_flag)
                    yb_tserver_cmd.append("--{}".format(complex_flag))
                else:
                    # Handle simple flags
                    yb_tserver_cmd.append("--{}".format(tserver_flag))

        if self.configs.saved_data.get("ysql_enable_auth"):
            yb_tserver_cmd.extend(["--ysql_enable_auth=true"])

        if self.configs.saved_data.get("use_cassandra_authentication"):
            yb_tserver_cmd.extend(["--use_cassandra_authentication=true"])

        if self.configs.saved_data.get("dns_enabled"):
            yb_tserver_cmd.extend(["--use_node_hostname_for_local_tserver=true"])

        return yb_tserver_cmd

    # Returns the ybc-server start command
    def get_ybc_cmd(self, data_dir):

        # yb-controller-server requires the log directory to be
        # created before hand.
        ybc_server_logdir = "{}/yb-data/ybc/logs".format(self.configs.saved_data.get("data_dir"))
        if not os.path.exists(ybc_server_logdir):
            os.makedirs(ybc_server_logdir, 0o755)
            if not os.path.exists(ybc_server_logdir):
                Output.log("Error: YBC logs dir didn't get created.")


        advertise_ip = self.advertise_ip()

        ybc_cmd = [find_ybc_binary_location("yb-controller-server")] + \
            [
                "--server_address={}".format(advertise_ip),
                "--yb_tserver_address={}".format(advertise_ip),
                "--yb_master_address={}".format(advertise_ip),
                "--log_dir={}".format(ybc_server_logdir),
                "--yb_admin={}".format(find_binary_location("yb-admin")),
                "--yb_ctl={}".format(find_binary_location("yb-ctl")),
                "--ysql_dump={}".format(find_postgres_binary_location("ysql_dump")),
                "--ysql_dumpall={}".format(find_postgres_binary_location("ysql_dumpall")),
                "--ysqlsh={}".format(find_binary_location("ysqlsh")),
                "--ycqlsh={}".format(find_binary_location("ycqlsh")),
                "--redis_cli={}".format(find_binary_location("redis-cli")),
            ]

        if self.configs.saved_data.get("secure"):
            ybc_cmd.extend([
                "--cert_node_filename={}".format(advertise_ip),
                "--certs_dir_name={}".format(self.configs.saved_data.get("certs_dir")),
            ])

        return ybc_cmd


    # Returns (error string, yw_started).
    # yw_started is True if YW was actually started
    def maybe_start_yw(self, is_first_run, is_first_install):
        was_started = False
        err = None

        if not self.configs.saved_data.get("ui"):
            return (err, was_started)

        yw_process = self.processes.get("yugaware")
        yw_proxy = YugaWareProxy(self.advertise_ip(),
            self.configs.saved_data.get("webserver_port"))

        # Setup schema for play framework.
        if is_first_run and not self.is_yw_initialized():
            Output.log("Setting up admin console schema...")
            Output.init_animation("Preparing UI schema...")
            if not self.init_yw():
                #TODO: make this return to caller
                Output.log_error_and_exit("Failed to set up admin console schema...")

            Output.update_animation("UI schema ready")

        try:
            if is_first_run:
                Output.init_animation("Bringing up UI...")

            # Start YW process.
            if not yw_process.is_running():
                if not is_first_run:
                    Output.log(
                        "Webserver died unexpectedly. Restarting...", logging.ERROR)
                yw_process.start()
                was_started = True

            # After first run, do not attempt any more setup, just return.
            if not is_first_run:
                return (err, was_started)

            # Login with username/pwd, this tells us that YW server is up.
            err = self.wait_yw_login(yw_proxy, insecure=False)
            if err:
                return (err, was_started)

            # On first install run, always setup YW.
            # On a first run that is not first install,
            # check if setup still needs to complete.
            needs_setup = is_first_install
            if not needs_setup:
                # Login insecurely - if this fails, this likely means
                # YW wasn't fully setup the first time around.
                err = self.wait_yw_login(yw_proxy, insecure=True)
                if err:
                    Output.log("Unable to insecure login to YW: {}".format(err))
                    needs_setup = True

            if not needs_setup:
                return (err, was_started)

            # Set up login without username and password.
            err = yw_proxy.set_security("insecure")
            if err:
                return (err, was_started)

            # Verify login without username and password.
            err = yw_proxy.insecure_login()
            if err:
                return (err, was_started)
            yw_logged_in = True

            full_master_list = self.wait_get_all_masters(timeout=60)
            if not full_master_list:
                err = "Unable to find full master list for YW import"
                return (err, was_started)

            # Import the universe (or re-import it). Re-importing should be harmless.
            err = yw_proxy.import_universe(
                        ",".join(full_master_list),
                        self.master_port(),
                        self.configs.saved_data.get("universe_uuid"))
            if err:
                return (err, was_started)

            err = yw_proxy.set_landing_page(
                    self.configs.saved_data.get("universe_uuid"))
            if err:
                return (err, was_started)

            if is_first_run:
                Output.update_animation("UI ready")
            if is_first_run and yw_process.is_running() and self.alerts:
                yw_proxy.send_alerts(self.alerts)

        finally:
            if is_first_run:
                animation_status = Output.ANIMATION_FAIL if err else Output.ANIMATION_SUCCESS
                Output.update_animation("UI status", status=animation_status)
        return (err, was_started)

    # Pushes yugabyted script to background as a daemon. The process is not tied to a shell, but
    # it will not survive between machine restarts.
    def daemonize(self):
        def remove_handlers():
            if PY_VERSION < 3:
                handlers = [e for e in atexit._exithandlers if e[0] == self.kill_children]
                for handler in handlers:
                    atexit._exithandlers.remove(handler)
            else:
                atexit.unregister(self.kill_children)

        if os.fork():
            # Delete any custom exit handlers so daemon has full control.
            remove_handlers()

            # If parent is interrupted, kill the children as well. Note there is potentially a
            # window where the daemon hasn't created its pidfile yet and this will error out before
            # it can kill the daemon.
            self.set_signals(self.stop)

            # Keep the parent process alive until the daemon confirms yugabyted started properly.
            try:
                self.script.daemon_success.get(timeout=600)
            except queue.Empty as e:
                Output.print_and_log(
                    "Timed out trying to start {} daemon.".format(SCRIPT_NAME), logging.ERROR)
                self.stop()
            sys.exit()
        os.chdir(YUGABYTE_DIR)
        os.setsid()
        os.umask(0)
        if os.fork():
            remove_handlers()
            sys.exit()
        Output.log("Daemon grandchild process begins execution.")

    # Sets env variables needed for yugabyted start.
    def set_env_vars(self):
        # Sets YW metrics to use local database.
        os.environ["USE_NATIVE_METRICS"] = "true"

    # Returns true if the system has been configured for clock bound.
    # Runs `configure_clockbound.sh --validate` and returns true if it returns 0.
    def assert_system_configured_for_clockbound(self):
        configure_clockbound_path = find_binary_location("configure_clockbound.sh")
        cmd = ["bash", configure_clockbound_path, "--validate"]
        _, _, retcode = run_process(cmd)
        return retcode == 0

    # Runs post_install script for linux computers.
    def post_install_yb(self):
        if not sys.platform.startswith('linux'):
            return

        post_install_script_path = find_binary_location('post_install.sh')

        # If post_install.sh script is not found then we assume that
        # we are executing it in development mode, hence skip post_install.sh script
        # TODO(Sanket): Refactor the design to have yugabyted a way of knowing whether it
        # is an install env v/s dev
        if(post_install_script_path is None):
            return

        Output.log("Running the post-installation script {} (may be a no-op)".format(
                    post_install_script_path))
        process = subprocess.Popen(
                post_install_script_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        std_out, std_err = process.communicate()
        if process.returncode != 0:
            Output.log_error_and_exit(
                "Failed running {} (exit code: {}). Standard output:\n{}\n. "
                "Standard error:\n{}".format(
                    post_install_script_path, process.returncode, std_out, std_err))
        Output.log("Successfully ran the post-installation script.")


    # Initialize YW process. Creates all necessary tables. Returns false if init failed.
    def init_yw(self):
        # Create Play evolutions table. Required for YugaWare to start up properly.
        create_play_table = [
            os.path.join(YUGAWARE_BIN_DIR, "yugaware"),
            "-Dconfig.file=" + YUGAWARE_CONF,
            "-Dlog.override.path={}".format(self.configs.saved_data.get("log_dir"))
        ]
        Output.log("Initializing play tables...")
        run_process(create_play_table)
        Output.log("Done initializing play tables.")

        return True

    # Returns if yb-master and yb-tserver were properly initialized before.
    def is_yb_initialized(self):
        for info_file in ("master-info", "tserver-info", "tserver-info-cql"):
            if not os.path.exists(os.path.join(self.configs.saved_data.get("data_dir"), info_file)):
                return False
        return True

    # Returns if yugaware was properly initialized before.
    def is_yw_initialized(self):
        # Check Play evolutions table was created.
        Output.log("Checking play_evolutions table")
        list_tables_cmd = [find_binary_location("ysqlsh"), "-d", WEBSERVER_DB, "-c", r"\d"]
        out, err, ret_code = run_process(list_tables_cmd)
        Output.log("Finished checking play evolutions table")
        return not err and not ret_code and "play_evolutions" in out

    # Returns true if this master was found in the current list of masters
    def wait_master(self, master_addr):

        if (not self.processes.get("master").is_running()):
            Output.log("Failed waiting for yb-master... process died.", logging.ERROR)
            raise RuntimeError("process died unexpectedly.")

        cur_master_uuids = [ m[0] for m in YBAdminProxy.get_masters(master_addr) ]
        master_uuid = self.configs.saved_data.get("master_uuid")
        if not cur_master_uuids:
            raise RetryableError()
        return master_uuid in cur_master_uuids

    # Returns the current list of master UUIDs
    def get_master_uuids(self, master_addr):

        if (not self.processes.get("master").is_running()):
            Output.log("Failed waiting for yb-master... process died.", logging.ERROR)
            raise RuntimeError("process died unexpectedly.")

        cur_master_uuids = [ m[0] for m in YBAdminProxy.get_masters(master_addr) ]
        if not cur_master_uuids:
            raise RetryableError()
        return cur_master_uuids

    # Use the masters we know (ourselves and the join target) to discover the full cluster.
    # Retry until timeout in case the masters we know are still coming up.
    def wait_get_all_masters(self, timeout=180):
        Output.log("Waiting to get the full master addrs list from master")
        try:
            return retry_op(self.get_all_masters, timeout)
        except RuntimeError:
            Output.log("Failed to query for all masters. Exception: {}".format(
                traceback.format_exc()))
            return False

    # Use the masters we know (ourselves and the join target) to discover the full cluster.
    def get_all_masters(self):

        current_masters = self.configs.saved_data.get("current_masters")

        if current_masters:
            all_masters = [ m[1] for m in YBAdminProxy.get_masters(current_masters) ]
            Output.log("Got all masters: {}".format(all_masters))
            if all_masters:
                return all_masters

        join_ip = self.configs.saved_data.get("join")
        advertise_ip = self.advertise_ip()
        all_masters = None
        for master_ip in (join_ip, advertise_ip):
            if not master_ip:
                continue
            master_addr = "{}:{}".format(get_url_from_ip(master_ip),
                                        self.configs.saved_data.get("master_rpc_port"))
            all_masters = [ m[1] for m in YBAdminProxy.get_masters(master_addr) ]
            Output.log("Got all masters: {}".format(all_masters))
            if all_masters:
                return all_masters

        raise RetryableError()


    # Waits till the newly added master comes in the list of masters of cluster
    def wait_get_master(self, master, timeout=180):
        Output.log("Waiting for master with address {} to be added".format(master))
        try:
            return retry_op_with_argument(self.get_master_at_addrs, master, timeout)
        except RuntimeError:
            Output.log("Failed to find master {} in the masters list. Exception: {}".format(
                traceback.format_exc()))
            return False

    # Raises a Retryable error if the master is not in the list of master of the cluster
    def get_master_at_addrs(self, master):

        current_masters = self.configs.saved_data.get("current_masters")
        all_masters_and_roles = ""
        if current_masters:
            all_masters_and_roles = { urlparse("//" + m[1]).hostname: m[2]
                                    for m in YBAdminProxy.get_masters(current_masters) }
        if not all_masters_and_roles:
            join_ip = self.configs.saved_data.get("join")
            master_ip = join_ip if join_ip else self.advertise_ip()
            master_addr = "{}:{}".format(get_url_from_ip(master_ip),
                                            self.configs.saved_data.get("master_rpc_port"))
            all_masters_and_roles = { urlparse("//" + m[1]).hostname: m[2]
                                        for m in YBAdminProxy.get_masters(master_addr) }

        if master in all_masters_and_roles and all_masters_and_roles[master] == "FOLLOWER":
            return True

        raise RetryableError()

    # Get the current list of masters and leader master known to a tserver using
    # api/v1/masters endpoint of the tserver.
    def get_current_masters_and_leader_from_api(self, tserverIp, timeout=30):
        if not tserverIp:
            Output.log("Empty tserverIp passed to get_current_masters_and_leader_from_api")
            return ('','')
        tserver_addr = "{}:{}".format(get_url_from_ip(tserverIp),
                            self.configs.saved_data.get("tserver_webserver_port"))
        try:
            tserverMastersAPI = "http://{}/api/v1/masters".format(tserver_addr)
            Output.log("Trying to get masters information from {}".format(tserverMastersAPI) +
                        " (Timeout={})".format(timeout))
            response = urlopen(Request(tserverMastersAPI), timeout=timeout)
            Output.log("Got the response from {}. response: {}".format(tserverMastersAPI, response))

            jsonResponseFromTserver = json.load(response)
            dictOfAllNodes = jsonResponseFromTserver.get("master_server_and_type")

            currentMastersCSV = ""
            lenOfMasters = len(dictOfAllNodes)
            masterLeader = ""
            for node in dictOfAllNodes:
                if node["is_leader"]:
                    masterLeader = urlparse("//" + node["master_server"]).hostname
                currentMastersCSV += node["master_server"]
                lenOfMasters -= 1
                if lenOfMasters > 0:
                    currentMastersCSV += ","

            Output.log("Tserver {} returned the following ".format(tserverIp) +
                "set of the current masters {}.".format(currentMastersCSV) +
                " and master leader {}.".format(masterLeader),
                    logging.DEBUG)

            return (currentMastersCSV, masterLeader)

        except HTTPError as http_err:
            Output.log('HTTP error occurred while fetching current' +
                    'masters from tserver: {}', http_err)
            return ('','')
        except URLError as url_err:
            Output.log('URL error occurred while fetching current' +
                    'masters from tserver: {}', url_err)
            return ('','')
        except Exception as err:
            Output.log('Other error occurred while fetching current' +
                    'masters from tserver: {}', err)
            return ('','')

    # Get the current list of masters known to a tserver using api/v1/masters endpoint
    # of the tserver. Use the node address given with the --join flag or use the
    # advertise address if join flag is not provided.
    def get_current_masters_from_api(self, tserverIp):
        if not tserverIp:
            return ""
        tserver_addr = "{}:{}".format(get_url_from_ip(tserverIp),
                            self.configs.saved_data.get("tserver_webserver_port"))
        try:
            tserverMastersAPI = "http://{}/api/v1/masters".format(tserver_addr)
            response = urlopen(Request(tserverMastersAPI))
            jsonResponseFromTserver = json.load(response)
            dictOfAllNodes = jsonResponseFromTserver.get("master_server_and_type")

            currentMastersCSV = ""
            lenOfMasters = len(dictOfAllNodes)
            for node in dictOfAllNodes:
                currentMastersCSV += node["master_server"]
                lenOfMasters -= 1
                if lenOfMasters > 0:
                    currentMastersCSV += ","

            Output.log("Tserver {} returned the following ".format(tserverIp) +
                "set of the current masters {}.".format(currentMastersCSV),
                    logging.DEBUG)

            return currentMastersCSV

        except HTTPError as http_err:
            Output.log('HTTP error occurred while fetching current' +
                    'masters from tserver: {}', http_err)
            return ''
        except URLError as url_err:
            Output.log('URL error occurred while fetching current' +
                    'masters from tserver: {}', url_err)
            return ''
        except Exception as err:
            Output.log('Other error occurred while fetching current' +
                    'masters from tserver: {}', err)
            return ''

    # Waits till the newly removed master doesn't comes in the list of masters of cluster
    def wait_remove_master(self, master, timeout=180):
        Output.log("Waiting for master with address {} to be removed".format(master))
        try:
            return retry_op_with_argument(self.remove_master_at_addrs, master, timeout)
        except RuntimeError:
            Output.log("Newly removed master {} is still present in the masters list." +
                    " Exception: {}".format(traceback.format_exc()))
            return False

    # Raises a Retryable error if the newly removed master is in the list of master of the cluster
    def remove_master_at_addrs(self, master):
        join_ip = self.configs.saved_data.get("join")
        master_ip = join_ip if join_ip else self.advertise_ip()
        master_addr = "{}:{}".format(get_url_from_ip(master_ip),
                                        self.configs.saved_data.get("master_rpc_port"))

        all_masters = [ urlparse("//" + m[1]).hostname
                       for m in YBAdminProxy.get_masters(master_addr) ]
        if master not in all_masters:
            return True

        raise RetryableError()

    # Get Leader master of the cluster
    def get_leader_master(self):

        current_masters = self.configs.saved_data.get("current_masters")
        if current_masters:
            all_masters_info = YBAdminProxy.get_masters(current_masters)
            for master_info in all_masters_info:
                if master_info[2] == "LEADER":
                    return master_info[1]

        join_ip = self.configs.saved_data.get("join")
        master_ip = join_ip if join_ip else self.advertise_ip()
        master_addr = "{}:{}".format(get_url_from_ip(master_ip),
                                        self.configs.saved_data.get("master_rpc_port"))
        all_masters_info = YBAdminProxy.get_masters(master_addr)
        for master_info in all_masters_info:
            if master_info[2] == "LEADER":
                return master_info[1]

        Output.log_error_and_exit("Couldn't get Leader master")

    # Get all tservers details
    def get_all_tserver_info(self, master_hostport):
        try:
            leaderMasterURL = "http://{}/api/v1/tablet-servers".format(master_hostport)
            response = urlopen(Request(leaderMasterURL))
            jsonResponseFromMaster = json.load(response)
            return jsonResponseFromMaster
        except HTTPError as http_err:
            Output.log("HTTP Error occured while hitting the api endpoint " +
                "http://{}/api/v1/tablet-servers: {}".format(master_hostport, http_err))
            Output.log_error_and_exit(Output.make_red("ERROR:") + " Master node " +
                "present at {} is not reachable.".format(master_hostport))
        except Exception as err:
            Output.log("HTTP Error occured while hitting the api endpoint " +
                "http://{}/api/v1/tablet-servers: {}".format(master_hostport, err))
            Output.log_error_and_exit(Output.make_red("ERROR:") + " Master node " +
                "present at {} is not reachable.".format(master_hostport))

    # Get tserver addresses
    def get_tserver_addresses(self):
        leader_master = urlparse("//" + self.get_leader_master()).hostname
        leader_master_http_endpoint = "{}:{}".format(
            get_url_from_ip(leader_master),
            self.configs.saved_data.get("master_webserver_port"))
        all_tserver_info = self.get_all_tserver_info(leader_master_http_endpoint)

        tserver_addresses = []
        for uuid in all_tserver_info:
            tservers = all_tserver_info[uuid]
            for tserver in tservers:
                tserver_addresses.append(tserver)

        return tserver_addresses

    # Get node version
    def get_node_version(self, tserver_hostport):
        try:
            versionURL = "http://{}/api/v1/version".format(tserver_hostport)
            response = urlopen(Request(versionURL))
            jsonResponse = json.load(response)
            return jsonResponse.get("version_number")

        except HTTPError as http_err:
            Output.log_error_and_exit("HTTP error occurred while fetching version from " +
                                    "tserver {}: {}".format(tserver_hostport, http_err))
        except Exception as err:
            Output.log_error_and_exit("Other error occurred while fetching version from " +
                                    "tserver {}: {}".format(tserver_hostport, err))

    # Get all nodes version
    def get_all_nodes_version(self, tserver_addresses):
        node_versions = {}
        for tserver_addr in tserver_addresses:
            version = self.get_node_version(tserver_addr)
            node_versions[tserver_addr] = version

        return node_versions

    # Get highest nodes version
    def get_highest_version(self, node_versions):
        highest_version = ""
        for _, version in node_versions.items():
            if highest_version == "" or self.compare_versions(
                                    version, highest_version) == 1:
                highest_version = version

        return highest_version

    # Compare Versions
    def compare_versions(self, versionA, versionB):
        int_version_a = self.get_int_version(versionA)
        int_version_b = self.get_int_version(versionB)

        for a, b in zip(int_version_a, int_version_b):
            if a < b:
                return -1
            elif a > b:
                return 1

        return 0

    # Convert Version to Integer for comparision
    def get_int_version(self, version_number):
        parts = version_number.split(".")[:3]
        int_versions = []
        for part in parts:
            int_versions.append(int(part))

        return int_versions

    # Get mismatched nodes
    def verify_all_nodes_version(self, node_versions, highest_version):
        mismatched_nodes = []
        for node, version in node_versions.items():
            if self.compare_versions(version, highest_version) != 0:
                mismatched_nodes.append(urlparse("//" + node).hostname)

        return mismatched_nodes

    # Get all masters placement locations
    def get_all_nodes_locations(self, all_tserver_info, placement_uuid):
        dictOfAllNodes = all_tserver_info.get(placement_uuid)
        placementInfoOfEveryNode = {}
        for node in dictOfAllNodes:
            hostname = urlparse("//" + node).hostname
            cloudLocationOfHost = "{}.{}.{}".format(dictOfAllNodes[node]["cloud"],
                dictOfAllNodes[node]["region"], dictOfAllNodes[node]["zone"])
            placementInfoOfEveryNode[hostname] = cloudLocationOfHost

        return placementInfoOfEveryNode

    # Get the location of a master in accordance to the fault tolerance
    def get_cloud_location_per_fault_tolerance(self, master_location):
        fault_tolerance = self.configs.temp_data.get("fault_tolerance")
        master_location = master_location.split(".")
        ft_location = ""
        if fault_tolerance == "cloud":
            ft_location = master_location[0]
        elif fault_tolerance == "region":
            ft_location = ".".join(master_location[:2])
        else:
            ft_location = ".".join(master_location)

        return ft_location

    # Get new valid master addresses that does not violate fault tolerance policy
    def get_new_valid_masters(self, current_masters, leader_master,
                              master_locations):

        replication_factor = int(self.configs.temp_data.get("replication_factor"))
        # ft_locations_to_masters_map is map in the form of {ft_location: list_of_masters}.
        # Here ft_location means the fault tolerance location (cloud.region for ft=region and
        # cloud.region.zone for ft=zone).
        # list_of_masters is the list of all the masters present in that ft_location.
        # This map is used to equally distribute the nodes if the number of ft_locations
        # is less than the replication factor (implied or explicit) specified in the
        # configure command.
        ft_locations_to_masters_map = dict()
        for master, location in master_locations.items():
            ft_location = self.get_cloud_location_per_fault_tolerance(location)
            if ft_location not in ft_locations_to_masters_map.keys():
                ft_locations_to_masters_map[ft_location] = [master]
            else:
                ft_locations_to_masters_map[ft_location].append(master)

        new_master_locations = dict()
        ft_location_of_leader_master = self.get_cloud_location_per_fault_tolerance(
                                                master_locations[leader_master])
        new_master_locations[leader_master] = ft_location_of_leader_master
        ft_locations_to_masters_map[ft_location_of_leader_master].remove(leader_master)
        # check if the current master's cloud location are already
        # statisfying the fault tolerance value and replication factor (implied or explicit)
        # specified in the configure command
        for master in current_masters:
            ft_location = self.get_cloud_location_per_fault_tolerance(master_locations[master])
            if ft_location not in new_master_locations.values():
                new_master_locations[master] = ft_location
                ft_locations_to_masters_map[ft_location].remove(master)

        if len(new_master_locations) == replication_factor:
            return list(new_master_locations.keys())

        # if new masters list is not equal to replication factor,
        # determine new masters based on the distinct cloud locations
        # of nodes in the cluster
        for master, location in master_locations.items():
            ft_location = self.get_cloud_location_per_fault_tolerance(location)
            if ft_location not in new_master_locations.values():
                new_master_locations[master] = ft_location
                ft_locations_to_masters_map[ft_location].remove(master)
            if len(new_master_locations) == replication_factor:
                return list(new_master_locations.keys())

        # if new master list is not equal to replication factor after iterating
        # through the distinct cloud locations, add the existing masters back
        # to the new master's list for satisfying the replication factor.
        new_valid_masters_list = list(new_master_locations.keys())
        while len(new_valid_masters_list) != replication_factor:
            for location, master_list in ft_locations_to_masters_map.items():
                if len(master_list) != 0:
                    new_valid_masters_list.append(master_list.pop())
                if len(new_valid_masters_list) == replication_factor:
                    break

        return new_valid_masters_list

    # Replace the old master with a new one to maintain fault tolerance policy
    def replace_master(self, master_addr, old_master, new_master):
        self.add_master_for_data_placement(master_addr, new_master)
        self.remove_master_for_data_placement(master_addr, old_master)

    # add a new master computed during configure data_placement command
    def add_master_for_data_placement(self, master_addr, new_master):

        new_master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        Output.log("Adding master {} to the cluster.".format(new_master))
        if not YBAdminProxy.add_master(master_addr, new_master, new_master_rpc_port):
            Output.log_error_and_exit("Failed to add master {} to the cluster."
                            .format(new_master))
        if not self.wait_get_master(new_master):
            Output.log_error_and_exit("Couldn't get master {} in the cluster masters list."
                            .format(new_master))
        else:
            Output.log("Added master {} to the cluster".format(new_master))

    # remove an master computed during configure data_placement command
    def remove_master_for_data_placement(self, master_addr, old_master):

        old_master_rpc_port = self.configs.saved_data.get("master_rpc_port")
        Output.log("Removing master {} from the cluster.".format(old_master))
        if not YBAdminProxy.remove_master(master_addr, old_master, old_master_rpc_port):
            Output.log_error_and_exit("Failed to remove master {} from the cluster."
                            .format(old_master))
        if not self.wait_remove_master(old_master):
            Output.log_error_and_exit("master {} in the cluster masters list still exists."
                            .format(old_master))
        else:
            Output.log("Removed master {} from the cluster".format(old_master))

    def parse_constraint_value(self, constraint_str):
        placement_constraints = []
        priority_constraints = []
        for item in constraint_str.split(","):
            parts = item.rsplit(":", 1)
            placement_constraints.append(parts[0])
            if len(parts) > 1:
                priority_constraints.append(item)

        placement_str = ",".join(placement_constraints)
        return placement_str, priority_constraints

    def enable_pitr(self, pitr_object, pitr_yb_api_type, schedules):
        pitr_object_name = pitr_object.split('.', 1)[1]
        pitr_object_exists, is_pitr_enabled = self.validate_pitr_configs(pitr_object, schedules)
        if not pitr_object_exists:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": The specified {} {} does not exist. Please "
            "specify a valid {} to enable point-in-time recovery.".format(
                                    pitr_yb_api_type, pitr_object_name, pitr_yb_api_type)
            )
        if is_pitr_enabled:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                    status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": Point-in-time recovery already enabled"
            " for {} {}. Please disable point-in-time recovery"
            " before attempting to enable it.".format(pitr_yb_api_type, pitr_object_name)
            )
        Output.update_animation("Verified Point-In-Time Recovery configs.")
        # Run YBAdmin command to enable pitr
        retention_period = self.configs.temp_data.get("retention_period")
        master_addresses = self.configs.saved_data.get("current_masters")
        Output.init_animation("Enabling Point-In-Time Recovery...")
        start_time = time.time()
        success = YBAdminProxy.create_snapshot_schedule(master_addresses,
            retention_period, pitr_object)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        if not success:
            Output.log_error_and_exit(Output.update_animation("Failed to "
                "enable Point-In-Time Recovery.",status=Output.ANIMATION_FAIL))
        else:
            Output.log("Successful in enabling point-in-time recovery.")

        Output.update_animation("Successfully enabled Point-In-Time recovery for {} {}.".format(
                                                            pitr_yb_api_type, pitr_object_name))
        # Send callhome event for PITR enable
        self.start_callhome_thread(
            CALLHOME_EVENT_PITR_ENABLE,
            lambda: self._send_pitr_enable_callhome(pitr_yb_api_type,
                                                     retention_period,
                                                     success, time_taken))
        self.shutdown_callhome_thread(CALLHOME_EVENT_PITR_ENABLE, timeout=5)
        status_details = self.get_pitr_config_details(pitr_object, retention = retention_period)

        Output.print_out(self.get_status_string_common(status_details[0], status_details[1]))

    def disable_pitr(self, pitr_object, pitr_yb_api_type, schedules):
        pitr_object_name = pitr_object.split('.', 1)[1]
        pitr_object_exists, is_pitr_enabled = self.validate_pitr_configs(pitr_object, schedules)
        if not pitr_object_exists:
            Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": The specified {} {} does not exist. Please "
            "specify a valid {} to disable point-in-time recovery.".format(
                                        pitr_yb_api_type, pitr_object_name, pitr_yb_api_type)
            )
        if not is_pitr_enabled:
            Output.update_animation("Failed to verify Point-In-Time Rcovery configs.",
                        status=Output.ANIMATION_FAIL)
            Output.log_error_and_exit(
            Output.make_red("ERROR") + ": Point-in-time recovery is not enabled"
            " for {} {}. Please enable point-in-time recovery"
            " before attempting to disable it.".format(pitr_yb_api_type, pitr_object_name)
            )
        Output.update_animation("Verified Point-In-Time Recovery configs.")
        # Get the schedule ID to disable pitr
        pitr_object_id, _, _ = self.get_pitr_object_details(pitr_object, schedules)
        master_addresses = self.configs.saved_data.get("current_masters")
        Output.init_animation("Disabling Point-In-Time Recovery...")
        start_time = time.time()
        success = YBAdminProxy.delete_snapshot_schedule(master_addresses, pitr_object_id)
        end_time = time.time()
        time_taken = self._format_time_taken_ms(start_time, end_time)
        if not success:
            Output.log_error_and_exit(Output.update_animation("Failed to "
                "disable Point-In-Time Recovery.",status=Output.ANIMATION_FAIL))
        else:
            Output.log("Successful in disabling point-in-time recovery.")

        Output.update_animation("Successfully disabled Point-In-Time recovery for {} {}.".format(
                                                            pitr_yb_api_type, pitr_object_name))
        # Send callhome event for PITR disable
        self.start_callhome_thread(
            CALLHOME_EVENT_PITR_DISABLE,
            lambda: self._send_pitr_disable_callhome(pitr_yb_api_type,
                                                     success, time_taken))
        self.shutdown_callhome_thread(CALLHOME_EVENT_PITR_DISABLE, timeout=5)
        status_details = self.get_pitr_config_details(pitr_object)

        Output.print_out(self.get_status_string_common(status_details[0], status_details[1]))


    def display_pitr(self, pitr_object, pitr_yb_api_type, schedules):

        status_details = []
        status_display_info = {}

        # Case Scenario: When either database or keyspace is specified
        if pitr_object:
            pitr_object_name = pitr_object.split('.', 1)[1]
            pitr_object_exists, is_pitr_enabled = self.validate_pitr_configs(pitr_object, schedules)
            if not pitr_object_exists:
                Output.update_animation("Failed to verify Point-In-Time Recovery configs.",
                        status=Output.ANIMATION_FAIL)
                Output.log_error_and_exit(
                Output.make_red("ERROR") + ": The specified {} {} does not exist. Please "
                "specify a valid {} to display point-in-time recovery status.".format(
                                pitr_yb_api_type, pitr_object_name, pitr_yb_api_type)
                )
            Output.update_animation("Verified Point-In-Time Recovery configs.")
            Output.init_animation("Fetching info from Point-In-Time Recovery configs...")
            if is_pitr_enabled:
                final_status = "Point-In-Time recovery enabled for {} {}.".format(
                                        pitr_yb_api_type, pitr_object_name)
                status_display_info[final_status] = Output.make_green
                # Get earliest recoverable time and retention
                _,earliest_recoverable_time, retention = self.get_pitr_object_details(
                                                            pitr_object, schedules)
                status_details = [
                (Output.make_yellow("Status"), final_status),
                ]
                status_details.append((Output.make_yellow("Recovery Schedule"),
                                        "Schedule:"))
                additional_status_details = self.display_pitr_helper(pitr_object,
                                                    earliest_recoverable_time, retention)
                status_details.extend(additional_status_details)
            else:
                final_status = "Point-In-Time recovery not enabled for {} {}.".format(
                                                        pitr_yb_api_type, pitr_object_name)
                status_details = [
                (Output.make_yellow("Status"), final_status),
                ]

        # Case Scenario: When neither database or keyspace is specified
        else:
            Output.update_animation("Verified Point-In-Time Recovery configs.")
            Output.init_animation("Fetching info from Point-In-Time Recovery configs...")
            active_schedules = False
            for schedule in reversed(schedules):
                options = schedule.get("options", {})
                if "delete_time" not in options:
                    active_schedules = True
                    break

            if active_schedules:
                final_status = "Point-In-Time recovery enabled."
                status_display_info[final_status] = Output.make_green
                status_details = [
                (Output.make_yellow("Status"), final_status),
                ]
                status_details.append((Output.make_yellow("Recovery Schedule"), "Following are the"
                                            " recovery schedules:"))

                # Display status detail for each schedule
                valid_schedule_count = 0
                for schedule in schedules:
                    options = schedule.get("options", {})
                    if options.get("delete_time"):
                        continue

                    valid_schedule_count += 1
                    pitr_object = options.get("filter", "")
                    retention = options.get("retention")
                    earliest_recoverable_time = schedule["snapshots"][0]["snapshot_time"]
                    schedule_entry = "Schedule " + str(valid_schedule_count) + ":"
                    status_display_info[schedule_entry] = Output.make_yellow
                    status_details.append((Output.make_yellow(""), schedule_entry))
                    additional_status_details = self.display_pitr_helper(pitr_object,
                                                    earliest_recoverable_time, retention)
                    status_details.extend(additional_status_details)

            else:
                final_status = "Point-In-Time recovery not enabled."
                status_details = [
                (Output.make_yellow("Status"), final_status),
                ]

        Output.update_animation("Successful in Fetching Point-In-Time Recovery configs.")
        Output.print_out(self.get_status_string_common(status_details, status_display_info))

    # Validate PITR configs
    def validate_pitr_configs(self, pitr_object, schedules):
        pitr_object_exists = False
        is_pitr_enabled = False
        pitr_object_name = pitr_object.split('.', 1)[1]
        # Validate database or keyspace
        if pitr_object.startswith("ysql."):
            ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
            pitr_object_exists = ysql_proxy.db_exists(pitr_object_name)
        else:
            ycql_proxy = YcqlProxy(self.advertise_ip(), self.configs.saved_data.get("ycql_port"))
            pitr_object_exists = ycql_proxy.keyspace_exists(pitr_object_name)
        # Check if pitr is already configured
        if pitr_object_exists:
            is_pitr_enabled = self.is_pitr_enabled_for_object(pitr_object, schedules)

        return pitr_object_exists, is_pitr_enabled

    # Check if PITR is enabled or not
    def is_pitr_enabled_for_object(self, pitr_object, schedules):
        # Check the most recent schedule
        for schedule in reversed(schedules):
            options = schedule.get("options", {})
            target_object = options.get("filter", "")
            if pitr_object == target_object:
                return not options.get("delete_time")

        return False

    # Get the schedule details
    def get_pitr_object_details(self, pitr_object, schedules):
        for schedule in reversed(schedules):
            options = schedule.get("options", {})
            target_object = options.get("filter", "")
            if pitr_object == target_object:
                schedule_id = schedule.get("id", " ")
                earliest_recoverable_time = schedule["snapshots"][0]["snapshot_time"]
                retention = options.get("retention")
                break

        return schedule_id, earliest_recoverable_time, retention

    # Print output for enabling, disabling and restoring pitr
    def get_pitr_config_details(self, pitr_object, retention = None, restore_time = None):
        status_details = []
        status_display_info = {}
        pitr_object_name = pitr_object.split('.', 1)[1]
        final_status = "Success"

        status_details = [
            (Output.make_yellow("Status"), final_status),
        ]
        status_display_info[final_status] = Output.make_green
        if pitr_object.startswith("ysql."):
            status_details.append((Output.make_yellow("Database"), pitr_object_name))
        else:
            status_details.append((Output.make_yellow("Keyspace"), pitr_object_name))

        if retention is not None:
            retention_days = "{} days".format(retention)
            status_details.append((Output.make_yellow("Retention Period"), retention_days))
            status_details.append((Output.make_yellow("Interval"), "24 hours"))

        if restore_time is not None:
            status_details.append((Output.make_yellow("Recovery Point"), restore_time))

        return [status_details, status_display_info]

    # Format PITR status details
    def display_pitr_helper(self, pitr_object, earliest_recoverable_time, retention):

        status_details = []
        # Parsing and formatting the earliest recoverable time and retention
        earliest_recoverable_time_object = \
            datetime.strptime(earliest_recoverable_time, "%Y-%m-%d %H:%M:%S.%f")
        earliest_recoverable_time_object = \
            (earliest_recoverable_time_object + timedelta(seconds=1)).replace(microsecond=0)
        retention_days = int(retention.split(' ')[0]) // 1440
        pitr_object_name = pitr_object.split('.', 1)[1]
        if pitr_object.startswith("ysql."):
            status_details.append((Output.make_yellow(""), "Database: " + pitr_object_name))
        else:
            status_details.append((Output.make_yellow(""), "Keyspace: " + pitr_object_name))
        status_details.append((Output.make_yellow(""), "Interval: " + "24 hours"))
        status_details.append((Output.make_yellow(""), "Retention: " +
                                                        str(retention_days) + " days"))
        status_details.append((Output.make_yellow(""), "Earliest Recoverable Time: "
                    + earliest_recoverable_time_object.strftime("%Y-%m-%d %I:%M:%S %p")))

        return status_details

    # Validate restore time
    def validate_restore_time(self, earliest_recoverable_time, restore_time):
        # Convert string times to datetime objects
        earliest_recoverable_time_object = \
            datetime.strptime(earliest_recoverable_time, "%Y-%m-%d %H:%M:%S.%f")
        earliest_recoverable_time_object = \
            (earliest_recoverable_time_object + timedelta(seconds=1)).replace(microsecond=0)
        restore_time_object = datetime.strptime(restore_time, "%Y-%m-%d %I:%M:%S %p")
        curr_time_object = datetime.now()

        # Restore time should be earlier than current time
        if restore_time_object > curr_time_object:
            error_msg = ("The specified restore time {} is in the future." + \
            " Please provide a time that is not later than the current time.").format(
                restore_time)
            return False, error_msg

        # Check if restore time is earlier than the earliest recoverable time
        if restore_time_object < earliest_recoverable_time_object:
            error_msg = ("The specified restore time {} is earlier than"
            " the earliest recoverable time. Please provide a time after {}.").format(
            restore_time, earliest_recoverable_time_object.strftime("%Y-%m-%d %I:%M:%S %p"))
            return False, error_msg

        # Restore time is valid
        return True, ""

    def get_configure_status_details(self, placement_locations, new_masters, priority_info):

        fault_tolerance = self.configs.temp_data.get("fault_tolerance")
        replication_factor = int(self.configs.temp_data.get("replication_factor"))

        # Get the number of nodes present in each AZ/region
        placement_location_map = dict()
        for master in new_masters:
            ft_location = self.get_cloud_location_per_fault_tolerance(placement_locations[master])
            if ft_location not in placement_location_map.keys():
                placement_location_map[ft_location] = 1
            else:
                placement_location_map[ft_location] += 1

        status_details = []
        status_display_info = {}

        # Case Scenario: When each AZ/region has only 1 node. (Happiest path)
        if len(placement_location_map) == replication_factor:
            final_status = ""
            ft_status = ""

            # Case Scenario: When rf = 1, the universe cannot survive the failure of the
            # AZ/region in which the master node is located.
            if replication_factor == 1:
                if fault_tolerance == "zone":
                    final_status = "Configuration successful."
                    ft_status = "Primary Cluster cannot survive even 1 availability zone failure"
                elif fault_tolerance == "region":
                    final_status = "Configuration successful. Primary data placement is " + \
                        "not geo-redundant."
                    ft_status = "Primary Cluster cannot survive even 1 region failure."

                status_details = [
                    (Output.make_yellow("Status"), final_status),
                    (Output.make_yellow("Fault Tolerance"), ft_status),
                ]
                status_display_info[ft_status] = Output.make_red

            # Case Scenario: When rf = 3, 5, 7. the universe can survive atmost (rf-1)/2
            # AZ/region failure
            else:
                if fault_tolerance == "zone":
                    final_status = "Configuration successful."
                    ft_status = "Primary Cluster can survive at most any " + \
                        "{} availability zone failure".format(replication_factor//2)
                elif fault_tolerance == "region":
                    final_status = "Configuration successful. Primary data placement is " + \
                        "geo-redundant."
                    ft_status = "Primary Cluster can survive at most any " + \
                        "{} region failure.".format(replication_factor//2)

                status_details = [
                    (Output.make_yellow("Status"), final_status),
                    (Output.make_yellow("Fault Tolerance"), ft_status),
                ]
                status_display_info[ft_status] = Output.make_green

        # Case Scenario: When all the master nodes belong to the same AZ/region
        # In this case, the universe cannot survive even 1 AZ/region failure.
        elif len(placement_location_map) == 1:
            final_status = ""
            ft_status = ""

            if fault_tolerance == "zone":
                final_status = "Configuration successful."
                ft_status = "Primary Cluster cannot survive even 1 availability zone failure."
            elif fault_tolerance == "region":
                final_status = "Configuration successful. Primary data placement is " + \
                    "not geo-redundant."
                ft_status = "Primary Cluster cannot survive even 1 region failure."

            status_details = [
                (Output.make_yellow("Status"), final_status),
                (Output.make_yellow("Fault Tolerance"), ft_status)
            ]
            status_display_info[ft_status] = Output.make_red

        # Case Scenario: When all master nodes are distributed accross 2 AZs/regions
        # In this case the universe can survive failure of only
        # 1 AZ/region (The one with lower number of master nodes).
        elif len(placement_location_map) == 2:
            number_of_failure_node_tolerance = replication_factor//2
            if fault_tolerance == "zone":
                final_status = "Configuration successful."
                ft_status = [
                    "For specified Replication factor (rf) {},".format(replication_factor) + \
                        " not enough availability zones.",
                    "Primary Cluster can survive the failure of only {} availability zone.".format(
                        [k for k, v in placement_location_map.items()
                            if v<=number_of_failure_node_tolerance][0]),
                    "Following are number of nodes in each availability zone."
                ]
            elif fault_tolerance == "region":
                final_status = "Configuration successful. Primary data placement is " + \
                    "not geo-redundant."
                ft_status = [
                    "For specified Replication factor (rf) {},".format(replication_factor) + \
                        " not enough regions.",
                    "Primary Cluster can survive the failure of only {} region.".format(
                        [k for k, v in placement_location_map.items()
                            if v<=number_of_failure_node_tolerance][0]),
                    "Following are number of nodes in each region."
                ]

            status_details = [
                (Output.make_yellow("Status"), final_status),
                (Output.make_yellow("Fault Tolerance"), ft_status[0]),
                (Output.make_yellow(""), ft_status[1]),
                (Output.make_yellow(""), ft_status[2]),
            ]
            status_display_info[ft_status[0]] = Output.make_yellow
            status_display_info[ft_status[1]] = Output.make_yellow

            for az, nodes in sorted(placement_location_map.items(), key=lambda x: x[1],
                                                                        reverse=True):
                az_node_info = "{} : {}".format(az, nodes)
                status_details += [
                    (Output.make_yellow(""), az_node_info)
                ]
                if nodes > replication_factor//2:
                    status_display_info[az_node_info] = Output.make_red
                else:
                    status_display_info[az_node_info] = Output.make_green

        # Case Scenario: When all master nodes are distributed accross more than 2 AZs/regions
        else:
            final_status = ""
            ft_status = ""
            if fault_tolerance == "zone":
                final_status = "Configuration successful."
                ft_status = [
                    "For specified Replication factor (rf) {},".format(replication_factor) + \
                        " not enough availability zones.",
                    "Primary Cluster can survive the failure of availability zones,",
                    "if the total number of down nodes in the YugabyteDB cluster doesn't " + \
                        "exceed {}.".format(replication_factor//2),
                    "Following are number of replicas in each availability zone."
                ]
            elif fault_tolerance == "region":
                final_status = "Configuration successful. Primary data placement is " + \
                    "not geo-redundant."
                ft_status = [
                    "For specified Replication factor (rf) {},".format(replication_factor) + \
                        " not enough regions.",
                    "Primary Cluster can survive the failure of regions,",
                    "if the total number of down nodes in the YugabyteDB cluster doesn't " + \
                        "exceed {}.".format(replication_factor//2),
                    "Following are number of replicas in each region."
                ]

            status_details += [
                (Output.make_yellow("Status"), final_status),
                (Output.make_yellow("Fault Tolerance"), ft_status[0]),
                (Output.make_yellow(""), ft_status[1]),
                (Output.make_yellow(""), ft_status[2]),
                (Output.make_yellow(""), ft_status[3]),
            ]
            status_display_info[ft_status[0]] = Output.make_yellow
            status_display_info[ft_status[1]] = Output.make_yellow
            status_display_info[ft_status[2]] = Output.make_yellow

            for az, nodes in sorted(placement_location_map.items(), key=lambda x: x[1],
                                    reverse=True):
                az_node_info = "{} : {}".format(az, nodes)
                status_details += [
                    (Output.make_yellow(""), az_node_info)
                ]

        if priority_info:
            preference_msg = "Successful in setting preference for zones."
            status_details.append((Output.make_yellow("Zone Preferences"), preference_msg))
            status_display_info[preference_msg] = Output.make_green
            status_details.append((Output.make_yellow(""),
                                   "Following are the preferences for zones"))
            sorted_priority_info = sorted(priority_info, key=lambda x: int(x.rsplit(':', 1)[1]))
            for item in sorted_priority_info:
                zone, priority = item.rsplit(':', 1)
                zone_preference_info = "{} : {}".format(zone, priority)
                status_details.append((Output.make_yellow(""), zone_preference_info))

        status_display_info = None if len(status_display_info) == 0 else status_display_info

        return [status_details, status_display_info]

    # This functions is used to determine if the placement constraint specified in
    # configure command is valid
    def is_placement_constraint_valid_values(self, placement_locations, placement_constraint):

        placement_constraint_list = placement_constraint.split(',')
        placement_locations_list = placement_locations.values()
        for constraint in placement_constraint_list:
            if constraint in placement_locations_list:
                is_valid = True
            else:
                is_valid = False
                break

        return is_valid

    def is_placement_constraint_valid_length(self, placement_constraint):

        replication_factor = int(self.configs.temp_data.get("replication_factor"))
        placement_constraint_list = placement_constraint.split(',')
        is_valid = True
        if len(placement_constraint_list) != replication_factor:
            is_valid = False

        return is_valid
    # az_to_num_rr_nodes_map is map in the form of {az: num_of_rr_nodes}.
    # Here az means the availability zone in form of cloud.region.zone.
    # num_of_rr_nodes is the number of all the nodes present in that az.
    def get_az_to_num_rr_nodes_map(self, placement_locations):
        az_to_num_rr_nodes_map = dict()
        for location in list(placement_locations.values()):
            if location not in az_to_num_rr_nodes_map.keys():
                az_to_num_rr_nodes_map[location] = 1
            else:
                az_to_num_rr_nodes_map[location] += 1

        return az_to_num_rr_nodes_map

    # Validates whether the new configuration provided by the user for modifying
    # rr cluster is different from the currently set configuration.
    def validate_new_config_for_rr_modify(self, cluster_config):
        rr_replication_factor = self.configs.temp_data.get("rr_replication_factor")
        rr_placement_constraint = self.configs.temp_data.get("rr_data_placement_constraint")

        old_rr_replication_factor = cluster_config["replicationInfo"] \
                                                ["readReplicas"][0].get("numReplicas")
        old_rr_placement_constraint = []
        for replica_info in cluster_config["replicationInfo"]["readReplicas"][0]. \
                                                                get("placementBlocks"):
            cloud_info = replica_info["cloudInfo"]
            old_rr_placement_constraint.append("{}.{}.{}:{}".format(cloud_info["placementCloud"],
                                                                  cloud_info["placementRegion"],
                                                                  cloud_info["placementZone"],
                                                                  replica_info["minNumReplicas"]))
        old_rr_placement_constraint = ",".join(old_rr_placement_constraint)

        def is_replication_factor_same():
            return int(rr_replication_factor) == old_rr_replication_factor

        def is_placement_constraint_same():
            rr_placement_constraint_list = rr_placement_constraint.split(',')
            old_rr_placement_constraint_list = old_rr_placement_constraint.split(',')
            placement_location_to_add = [location for location in rr_placement_constraint_list
                                            if location not in old_rr_placement_constraint_list]
            return len(placement_location_to_add) == 0

        if rr_replication_factor and is_replication_factor_same() and rr_placement_constraint \
                                                            and is_placement_constraint_same():
            Output.print_and_log(Output.make_red("ERROR") + ": You have provided the " +
                                        "same data_placement_constraint and replication_factor " +
                                        "as the ones that are currently set. Please provide " +
                                        "any new configuration to run " +
                                        "`yugabyted configure_read_replica modify` command.")
            sys.exit(1)
        elif rr_replication_factor and is_replication_factor_same():
            if rr_placement_constraint:
                msg = "You have provided the same replication_factor as the one that is " + \
                    "currently set."
            else:
                msg = "You have provided the same replication_factor as the one that is " + \
                    "currently set and no new data_placement_constraint was provided."

            msg += " Please provide any new configuration to " + \
                                    "run `yugabyted configure_read_replica modify` command."
            Output.print_and_log(Output.make_red("ERROR") + ": " + msg)
            sys.exit(1)
        elif rr_placement_constraint and is_placement_constraint_same():
            if rr_replication_factor:
                msg = "You have provided the same data_placement_constraint as the one that " + \
                    "is currently set."
            else:
                msg = "You have provided the same data_placement_constraint as the one that " + \
                    "is currently set and no new replication factor was provided."

            msg += " Please provide any new configuration to " + \
                                    "run `yugabyted configure_read_replica modify` command."
            Output.print_and_log(Output.make_red("ERROR") + ": " + msg)
            sys.exit(1)

    # Find/Validate the rr_replication_factor and data_placement_policy for read replica cluster
    def get_rr_rf_and_placement_constraint(self, placement_locations):
        rr_replication_factor = self.configs.temp_data.get("rr_replication_factor")
        rr_placement_constraint = self.configs.temp_data.get("rr_data_placement_constraint")

        az_to_num_rr_nodes_map = self.get_az_to_num_rr_nodes_map(placement_locations)

        if not rr_replication_factor:
            # CASE 1
            # When both rf and placement constraint was not provided by the user
            # rf = num of different zones
            if not rr_placement_constraint:
                rr_replication_factor = str(len(az_to_num_rr_nodes_map))
                rr_placement_constraint = self.get_rr_placement_constraint(az_to_num_rr_nodes_map)

            # CASE 2
            # When rf was not given but placement constraint was given
            # Validate the given data placement constraint against multiple checks
            # rf = total number of replicas mentioned in the constraint
            else:
                self.validate_rr_data_placement_constraint(az_to_num_rr_nodes_map,
                                                           rr_placement_constraint)

                # If all checks pass for the data plcamenet constraint, then extract rf from it.
                # rf = total number of replicas mentioned in the constraint.
                rr_replication_factor = str(self.extract_total_replicas_from_constraint(
                                                                        rr_placement_constraint))
        # CASE 3
        # When rf is given but placement constraint is not provided
        # Validate rf and extract data placement constraint
        elif not rr_placement_constraint:

                # CHECK 1
                # Check if the given rr_replication_faction is <= total num of read replica nodes.
                if len(placement_locations) < int(rr_replication_factor):
                    Output.print_and_log(msg = Output.make_red("ERROR") + ": Not enough nodes " +
                            "to set-up a RF-{} read replica cluster".format(rr_replication_factor),
                            level=logging.ERROR)
                    sys.exit(1)

                # CHECK 2
                # Check if the number of AZs which has read
                # replica nodes is <= rr_replication_factor
                if len(az_to_num_rr_nodes_map) > int(rr_replication_factor):
                    Output.print_and_log(msg = Output.make_red("ERROR") + ": There needs to be " +
                            "a replica in each AZ which contains a read replica node. There " +
                            "are {} AZs which hosts read ".format(len(az_to_num_rr_nodes_map)) +
                            "replica nodes whereas the replication factor provided " +
                            "was only {}".format(rr_replication_factor), level=logging.ERROR)
                    sys.exit(1)

                # If all checks pass for the provided replication factor,
                # then get the optimal data_placement_constraint
                rr_placement_constraint = self.get_rr_placement_constraint(az_to_num_rr_nodes_map)

        # CASE 4
        # When both rf and placemnet constraint is provided by the user
        # Validate the given data placement constraint against multiple checks
        # Validate rf with the num of replicas mentioned in the placement constraint
        else:
            self.validate_rr_data_placement_constraint(az_to_num_rr_nodes_map,
                                                       rr_placement_constraint)

            # CHECK 4
            # Check if the total number of replicas mentioned in the placement constraint is
            # equal to the replication factor provided by the user.
            if not self.is_total_num_of_replicas_valid_for_rr(rr_placement_constraint,
                                                                    rr_replication_factor):
                Output.log_error_and_exit(Output.make_red("FAILED") + ": The total number of " +
                                          "replicas mentioned in the placement constraint does " +
                                          "not match the given replication factor.")

        return [rr_replication_factor, rr_placement_constraint]

    # Validate data placement constraint provided by the user against multiple checks
    def validate_rr_data_placement_constraint(self, az_to_num_rr_nodes_map,
                                              rr_placement_constraint):

        # CHECK 1
        # Check if all the placement locations in which read replica nodes are present has
        # been provided by the user or not.
        az_not_present_in_constraint_list = self.are_all_locations_present_in_constraint(
                                        az_to_num_rr_nodes_map, rr_placement_constraint)

        if az_not_present_in_constraint_list:
            error_msg = Output.make_red("FAILED") + ": Invalid placement location " + \
                    "values provided for placement constraint. There needs to be " + \
                    "at least 1 replica in each AZ that contains a read replica node. "

            if len(az_not_present_in_constraint_list) == 1:
                error_msg += "The following AZ is not mentioned in placement constraint.\n"
            else:
                error_msg += "The following AZs are not mentioned in placement " + \
                    "constraint.\n"


            for az in az_not_present_in_constraint_list:
                error_msg += Output.make_red("* " + az) + "\n"

            Output.log_error_and_exit(error_msg)

        # CHECK 2
        # Check if the placement locations mentioned in the constraint mentioned by the
        # user has read replica nodes in them.
        az_in_constraint_not_present_list = self.is_placement_constraint_valid_values_for_rr(
                                            az_to_num_rr_nodes_map, rr_placement_constraint)

        if az_in_constraint_not_present_list:
            error_msg = Output.make_red("FAILED") + ": Invalid placement location " + \
                "values provided for placement constraint. Cloud locations mentioned " + \
                "in the placement constraint should contain read replica nodes."

            if len(az_in_constraint_not_present_list) == 1:
                error_msg += "The following AZ mentioned in placement constraint does " + \
                                                "not contain any read replica nodes.\n"
            else:
                error_msg += "The following AZs mentioned in placement constraint " + \
                                            "does not contain any read replica nodes.\n"


            for az in az_in_constraint_not_present_list:
                error_msg += Output.make_red("* " + az) + "\n"

            Output.log_error_and_exit(error_msg)

        # CHECK 3
        # Check if the nodes present for each cloud location is >=
        # number of replicas mentioned for that cloud location.
        invalid_num_replica_locations = self.is_num_replicas_for_each_az_valid_for_rr(
                                        az_to_num_rr_nodes_map, rr_placement_constraint)

        # print(invalid_num_replica_locations)
        if invalid_num_replica_locations:
            msg = Output.make_red("FAILED") + ": Invalid number of replicas mentioned.\n"


            for invalid_num_replica_location in invalid_num_replica_locations:

                if int(invalid_num_replica_location[1]) == 1:
                    msg += "* There is only 1 node in the " + \
                            "{} ".format(invalid_num_replica_location[0]) + \
                            "but {} ".format(invalid_num_replica_location[2]) + \
                            "replicas were mentioned.\n"
                else:
                    msg += "* There are only {} nodes".format(invalid_num_replica_location[1]) + \
                            " in the {}".format(invalid_num_replica_location[0]) + \
                            " but {} ".format(invalid_num_replica_location[2]) + \
                            "replicas were mentioned.\n"

            Output.log_error_and_exit(msg)

    # Deduce the optimal placement constraint for read replica cluster
    def get_rr_placement_constraint(self, az_to_num_rr_nodes_map):

        rr_replication_factor = self.configs.temp_data.get("rr_replication_factor")
        if not rr_replication_factor:
            rr_replication_factor = len(az_to_num_rr_nodes_map)
        else:
            rr_replication_factor = int(rr_replication_factor)

        az_to_num_replicas_map = dict()
        total_replicas = 0
        while total_replicas < rr_replication_factor:
            for location, num_nodes in az_to_num_rr_nodes_map.items():
                if num_nodes != 0:
                    if location not in az_to_num_replicas_map.keys():
                        az_to_num_replicas_map[location] = 1
                    else:
                        az_to_num_replicas_map[location] += 1

                    az_to_num_rr_nodes_map[location] -= 1
                    total_replicas += 1

                if total_replicas == rr_replication_factor:
                    break

        rr_placement_constraint = [ "{}:{}".format(location, num_of_replicas)
                          for location, num_of_replicas in
                          az_to_num_replicas_map.items()]

        rr_placement_constraint = ",".join(rr_placement_constraint)

        return rr_placement_constraint

    # Extracts rr_replication_factor from the data_placement_constraint mentioned by the user.
    def extract_total_replicas_from_constraint(self, placement_constraint):

        placement_constraint_list = placement_constraint.split(',')
        num_replicas_list = [ int(location_with_replicas.split(':')[1]) for
                             location_with_replicas in placement_constraint_list ]
        total_replicas = sum(num_replicas_list)

        return total_replicas

    # This functions is used to determine if the placement locations in which read replica
    # nodes are present has been provided by the user or not.
    def are_all_locations_present_in_constraint(self, az_to_num_rr_nodes_map,
                                                    rr_placement_constraint):

        placement_constraint_list = rr_placement_constraint.split(',')
        placement_constraint_list = [ location_with_replicas.split(':')[0] for
                                     location_with_replicas in placement_constraint_list ]

        placement_locations_list = list(az_to_num_rr_nodes_map.keys())

        az_not_present_list = list()

        for location in placement_locations_list:
            if location not in placement_constraint_list:
                az_not_present_list.append(location)

        return None if len(az_not_present_list) == 0 else \
                                        az_not_present_list

    # This functions is used to determine if the placement locations mentioned in the constraint
    # mentioned by the user has read replica nodes in them.
    def is_placement_constraint_valid_values_for_rr(self, az_to_num_rr_nodes_map,
                                                    rr_placement_constraint):
        placement_constraint_list = rr_placement_constraint.split(',')
        placement_constraint_list = [ location_with_replicas.split(':')[0] for
                                     location_with_replicas in placement_constraint_list ]

        placement_locations_list = list(az_to_num_rr_nodes_map.keys())

        az_not_present_list = list()

        for location in placement_constraint_list:
            if location not in placement_locations_list:
                az_not_present_list.append(location)

        return None if len(az_not_present_list) == 0 else \
                                        az_not_present_list

    # Checks if the total number of replicas mentioned in the placement constraint is
    # equal to the replication factor provided by the user.
    def is_total_num_of_replicas_valid_for_rr(self, placement_constraint,
                                                      rr_replication_factor):
        total_replicas = self.extract_total_replicas_from_constraint(placement_constraint)
        is_valid = True
        if total_replicas != int(rr_replication_factor):
            is_valid = False

        return is_valid

    # Checks if the nodes present for each cloud location is at least equal or more
    # than the number of replicas mentioned for that cloud location.
    def is_num_replicas_for_each_az_valid_for_rr(self, az_to_num_rr_nodes_map,
                                                 placement_constraint):
        placement_constraint_list = placement_constraint.split(',')

        invalid_num_replica_locations = list()
        for location_with_replicas in placement_constraint_list:
            location_with_replicas = location_with_replicas.split(':')
            location, replicas = location_with_replicas[0], location_with_replicas[1]
            # print(location, replicas)
            if az_to_num_rr_nodes_map.get(location) < int(replicas):
                invalid_num_replica_locations.append([location,
                                                str(az_to_num_rr_nodes_map.get(location)),
                                                replicas])
        # print(invalid_num_replica_locations)
        return invalid_num_replica_locations if \
                        len(invalid_num_replica_locations) > 0 else None

    # Get Outbound xcluster replications' statuses
    def get_outbound_xcluster_replication_statuses(self, xcluster_api_response,
                                                                replication_id = None):
        outbound_replication_groups = xcluster_api_response.get("outbound_replication_groups")
        if len(outbound_replication_groups) == 0:
            Output.log("No outbound replication found for target cluster.")
            return ""

        status_details = []
        status_display_info = {}

        replication_id_with_databases = self.get_databases_data_for_xcluster()
        if len(replication_id_with_databases) == 0:
            Output.log("Cannot retreive the databases for " +
                                                    "replication_id {}.".format(replication_id))

        replication_count = 1
        found_xcluster_status = False
        for replication_group in outbound_replication_groups:
            if (not replication_id) or (replication_id and \
                                replication_id == replication_group.get("replication_group_id")):
                status_details.extend([
                    (Output.make_yellow("Replication {}".format(replication_count)), ""),
                    (Output.make_yellow("Replication ID"),
                                                replication_group.get("replication_group_id")),
                ])

                replication_metadata = replication_group.get("metadata")
                namespace_infos = replication_metadata.get("namespace_infos")
                target_universe_info = replication_metadata.get("target_universe_info")
                if target_universe_info:
                    status_details.extend([
                        (Output.make_yellow("State"), target_universe_info.get("state")),
                        (Output.make_yellow("Target Universe UUID"),
                                                    target_universe_info.get("universe_uuid")),
                    ])
                else:
                    status_details.append(
                        (Output.make_yellow("State"), "CHECKPOINT")
                    )

                status_details.extend([
                    (Output.make_yellow("Databases"), "Following are the databases included " +
                                                            "in this replication"),
                ])

                databases_info = []

                for replication_group_info in replication_id_with_databases:
                    if replication_group_info["replication_id"] == \
                            replication_group.get("replication_group_id"):
                        databases_info = replication_group_info.get("databases")


                db_count = 1
                for db_info in databases_info:
                    for namespace_info in namespace_infos:
                        if db_info.get("id") == namespace_info.get("key"):
                            status_details.extend([
                                (Output.make_yellow(""), "Database {}:".format(db_count)),
                                (Output.make_yellow(""),
                                    "Name: {}".format(db_info.get("name"))),
                                (Output.make_yellow(""),
                                    "State: {}".format(namespace_info.get("value").get("state"))),
                            ])
                            if not status_display_info.get("Database {}:".format(db_count)):
                                status_display_info["Database {}:".format(db_count)] = \
                                                                                Output.make_yellow
                            db_count += 1

                replication_count += 1

                found_xcluster_status = True

        if not found_xcluster_status:
            if replication_id:
                Output.log("Couldn't find any outbound " +
                    "replication with replication id {}".format(replication_id))
            else:
                Output.log("Couldn't find any outbound replication groups.")
            return ""

        return self.get_status_string_common(status_details, status_display_info)

    # Get Inbound xcluster replications' statuses
    def get_inbound_xcluster_replication_statuses(self, xcluster_api_response, master_addresses,
                                                  replication_id = None):
        replication_infos = xcluster_api_response.get("replication_infos")
        if len(replication_infos) == 0:
            Output.log("No inbound replication found for target cluster.")
            return ""

        status_details = []
        status_display_info = {}

        xcluster_safe_time_statuses = YBAdminProxy.get_xcluster_safe_time(master_addresses)
        if xcluster_safe_time_statuses is None:
            Output.log("Couldn't get xcluster safe time status.")
            return ""

        if len(xcluster_safe_time_statuses) == 0:
            Output.log("No inbound xcluster replication found. Set-up xcluster " +
                        "replication using `xcluster create_checkpoint` and `xcluster set_up`, " +
                        "then run `xcluster status` on the target node to get status of " +
                                 "replication.")
            return ""

        replication_count = 1
        found_xcluster_status = False
        for replication_info in replication_infos:
            if (not replication_id) or (replication_id and \
                                replication_id == replication_info.get("replication_group_id")):
                status_details.extend([
                    (Output.make_yellow("Replication {}".format(replication_count)), ""),
                    (Output.make_yellow("Replication ID"),
                                                replication_info.get("replication_group_id")),
                    (Output.make_yellow("State"), replication_info.get("state")),
                ])

                source_nodes = [producer.get("host") for producer in
                                replication_info.get("producer_master_addresses")[1:]]

                status_details.extend([
                    (Output.make_yellow("Source cluster nodes"), ','.join(source_nodes)),
                    (Output.make_yellow("Databases"), "Following are the databases included " +
                                                            "in this replication"),
                ])

                databases_consumer_ids = [namespace_info.get("consumer_namespace_id") for
                    namespace_info in replication_info.get("db_scoped_info").get("namespace_infos")]

                db_count = 1
                for db_consumer_id in databases_consumer_ids:
                    for xcluster_status in xcluster_safe_time_statuses:
                        if db_consumer_id == xcluster_status.get("namespace_id"):
                            status_details.extend([
                                (Output.make_yellow(""), "Database {}:".format(db_count)),
                                (Output.make_yellow(""),
                                        "Name: {}".format(xcluster_status.get("namespace_name"))),
                                (Output.make_yellow(""),
                                        "Safe Time: {}".format(xcluster_status.get("safe_time"))),
                                (Output.make_yellow(""),
                                        "Safe Time Lag(micro secs): {}".format(
                                            xcluster_status.get("safe_time_lag_sec"))),
                                (Output.make_yellow(""),
                                        "Safe Time Skew(micro secs): {}".format(
                                            xcluster_status.get("safe_time_skew_sec"))),
                            ])
                            if not status_display_info.get("Database {}:".format(db_count)):
                                status_display_info["Database {}:".format(db_count)] = \
                                                                                Output.make_yellow
                            db_count += 1

                replication_count += 1

                found_xcluster_status = True

        if not found_xcluster_status:
            if replication_id:
                Output.log("Couldn't find any inbound " +
                    "replication with replication id {}".format(replication_id))
            else:
                Output.log("Couldn't find any inbound replication groups.")
            return ""

        return self.get_status_string_common(status_details, status_display_info)

    # Updating the database user passwords for postgres and cassandra
    def update_db_passwords(self, new_password):
        ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
        Output.log("Attempting to update Postgres password.")
        try:
            retry_op_with_argument(ysql_proxy.try_update_password, new_password, timeout=60)
        except RuntimeError:
            Output.log_error_and_exit("Could not update Postgress user password." +
                " Exception: {}".format(traceback.format_exc()))

        self.setup_env_init.setup_cert_file_path(self.configs.saved_data.get("ca_cert_file_path"))
        ycql_proxy = YcqlProxy(self.advertise_ip(), self.configs.saved_data.get("ycql_port"),
                        secure=self.configs.saved_data.get("secure"))
        Output.log("Attempting to update Cassandra password.")
        try:
            retry_op_with_argument(ycql_proxy.try_update_password, new_password, timeout=60)
        except RuntimeError:
            Output.log_error_and_exit("Could not update Cassandra user password." +
                " Exception: {}".format(traceback.format_exc()))

        self.setup_env_init.update_passwords(new_password)
        self.configs.saved_data["database_password"] = new_password

    # Create a new file in base directory and put Ysql login credentials in it.
    def create_password_file(self):
        password_file = os.path.join(self.configs.saved_data.get("data_dir"),
                "{}_credentials.txt".format(SCRIPT_NAME))

        ysql_username, ysql_password, ysql_db = self.setup_env_init.get_ysql_credentials()
        ycql_username, ycql_password, ycql_keyspace = self.setup_env_init.get_ycql_credentials()

        content = ""
        content += "YSQL Credentials:\nUsername: {}\nPassword: {}\n" \
            "Database: {}\n\n".format(ysql_username, ysql_password, ysql_db)
        content += "YCQL Credentials:\nUsername: {}\nPassword: {}\n" \
            "Keyspace: {}\n".format(ycql_username, ycql_password, ycql_keyspace)

        with open(password_file, "w+") as f:
            f.write(content)

    # When the 3rd node joins, this method will set the placement location and rf to 3
    def setup_first_cluster_config(self):
        placement_uuid = self.configs.saved_data.get("placement_uuid")
        leader_master = urlparse("//" + self.get_leader_master()).hostname
        leader_master_http_endpoint = "{}:{}".format(get_url_from_ip(leader_master),
                        self.configs.saved_data.get("master_webserver_port"))

        all_tserver_info = self.get_all_tserver_info(leader_master_http_endpoint)
        placement_locations = self.get_all_nodes_locations(all_tserver_info, placement_uuid)
        placement_info = list(placement_locations.values())
        placement_info.append("{}.{}.{}".format(self.configs.saved_data.get("cloud_provider"),
                                                self.configs.saved_data.get("cloud_region"),
                                                self.configs.saved_data.get("cloud_zone")))
        placement_info = ",".join(placement_info)

        master_addresses = list(placement_locations.keys())
        master_addresses.append(self.configs.saved_data.get("advertise_address"))
        master_addresses = ["{}:{}".format(get_url_from_ip(master),
                                           self.configs.saved_data.get("master_rpc_port")) \
                                           for master in master_addresses]
        master_addresses = ",".join(master_addresses)

        if not YBAdminProxy.modify_placement_info(master_addresses,
                                                  placement_info, placement_uuid):
            Output.log_error_and_exit(Output.make_red("FAILED") + ": Setting of default " +
                                      "data placement policy on the cluster.")

    # Verify that the master is in the current list of masters.
    # If not, set it up appropriately.
    def setup_master(self, timeout=180):
        Output.log("Waiting for master")
        join_ip = self.configs.saved_data.get("join")
        master_addrs = self.get_current_masters_from_api(join_ip)
        # API didn't return any masters addresses, use join or advertise address
        if not master_addrs:
            join_ip = self.configs.saved_data.get("join")
            master_ip = join_ip if join_ip else self.advertise_ip()
            master_addrs = "{}:{}".format(get_url_from_ip(master_ip),
                                        self.configs.saved_data.get("master_rpc_port"))

        # save the current known masters received from the get_current_masters_from_api()
        self.configs.saved_data["current_masters"]= master_addrs

        try:
            master_uuids = retry_op_with_argument(self.get_master_uuids, master_addrs)
            current_node_master_uuid = self.configs.saved_data.get("master_uuid")
            # If False, it means the master is
            # not part of the current set of masters. If we have a
            # join_ip, let's try to add ourselves to it, otherwise
            # it is a hard failure.
            if current_node_master_uuid in master_uuids:
                if not join_ip:
                    placement_uuid = self.configs.saved_data.get("placement_uuid")
                    placement_info = list()
                    placement_info = "{}.{}.{}".format(
                                                self.configs.saved_data.get("cloud_provider"),
                                                self.configs.saved_data.get("cloud_region"),
                                                self.configs.saved_data.get("cloud_zone"))
                    advertise_address = self.configs.saved_data.get("advertise_address")
                    master_addresses = "{}:{}".format(get_url_from_ip(advertise_address),
                                                self.configs.saved_data.get("master_rpc_port"))

                    if not YBAdminProxy.modify_placement_info(master_addresses, placement_info,
                                                        placement_uuid, replication_factor="1"):
                        Output.log_error_and_exit(Output.make_red("FAILED") + ": Setting of " +
                                                "default data placement policy on the cluster.")
                    return True
            if not join_ip:
                return False
        except RuntimeError:
            Output.log_error_and_exit("Failed to setup master. Exception: {}".format(
                traceback.format_exc()))
            return False

        # The master was not in the current list of masters
        # and we have a valid join_ip
        advertise_ip = self.advertise_ip()
        master_rpc_port = self.configs.saved_data.get("master_rpc_port")

        if len(master_uuids) >= 3:
            # this is going to be a standalone shell master
            return True
        if not YBAdminProxy.add_master(master_addrs, advertise_ip, master_rpc_port):
            Output.log_error_and_exit("Unable to add master {} to existing cluster at {}.".format(
                advertise_ip, join_ip))
            return False

        # If we are the third master, set replication factor to 3. This makes the cluster
        # automatically expand to rf3 when the third node is added.
        if len(master_uuids) == 2:
            self.setup_first_cluster_config()

        try:
            if retry_op_with_argument(self.get_master_uuids, master_addrs, timeout):
                Output.log("Completed setup and wait for master.")
                return True
        except RuntimeError:
            Output.log("setup_master: exception: {}".format(traceback.format_exc()))

        Output.log("Failed to setup master, did add_master succeed?")
        return False

    def is_tserver_up(self, cluster_type):
        join_ip = self.configs.saved_data.get("join")

        if cluster_type == "primary":
            master_addr = self.configs.saved_data.get("current_masters")
        else:
            master_addr = self.get_current_masters_from_api(join_ip)

        if not master_addr:
            master_ip = join_ip if join_ip else self.advertise_ip()
            master_addr = "{}:{}".format(get_url_from_ip(master_ip),
                                     self.configs.saved_data.get("master_rpc_port"))
        self.configs.saved_data["current_masters"] = master_addr

        if (cluster_type == "primary" and not self.processes.get("master").is_running()):
            Output.log("Failed waiting for yb-master... process died.", logging.ERROR)
            raise RuntimeError("process not running")

        if not self.processes.get("tserver").is_running():
            Output.log("Failed waiting for yb-tserver... process died.", logging.ERROR)
            raise RuntimeError("process not running")

        cur_tservers = YBAdminProxy.get_tservers(master_addr)
        tserver_uuid = self.configs.saved_data.get("tserver_uuid")
        if not cur_tservers or tserver_uuid not in cur_tservers:
            raise RetryableError()

        Output.log("Found tserver uuid {} in list of tserver uuids {}.".format(
            tserver_uuid, cur_tservers))
        return True

    def wait_tserver(self, cluster_type, timeout=180):
        Output.log("Waiting for tserver ...")
        try:
            if retry_op_with_argument(self.is_tserver_up, cluster_type, timeout):
                Output.log("Completed waiting for tserver.")
                return True
        except RuntimeError:
            Output.log("wait_tserver: exception: {}".format(traceback.format_exc()))

        Output.log("Failed to wait for tserver.")
        return False

    # In a multi-node cluster, the tserver initially knows just about its own master and the
    # master it is joining. After the cluster is formed, this method will attempt to
    # refresh the full list of masters so that the tserver can become aware of other masters.
    def update_tserver_master_addrs(self):
        tserver_cmd = self.processes["tserver"].cmd
        master_flag = [ flag for flag in tserver_cmd if flag.find(TS_MASTER_ADDRS_FLAG) >= 0 ]
        full_master_list = self.configs.saved_data.get("current_masters").split(",")

        if not full_master_list:
            Output.log("Unable to get all masters list, keeping tserver flag: {}".format(
                master_flag))
            return

        if len(full_master_list) < 3:
            my_master_ip = "{}:{}".format(get_url_from_ip(self.advertise_ip()),
                                          self.configs.saved_data.get("master_rpc_port"))
            full_master_list = list(set(full_master_list + [ my_master_ip ]))
            Output.log("Got full master addrs list: {}".format(full_master_list))

        new_master_flag = "--{}={}".format(TS_MASTER_ADDRS_FLAG, ",".join(full_master_list))
        Output.log("Old master flag is: {} and new master flag is: {}".format(
            master_flag, new_master_flag))
        if master_flag:
            tserver_cmd.remove(master_flag[0])
        tserver_cmd += [new_master_flag]
        self.configs.saved_data["current_masters"] = ",".join(full_master_list)

    def wait_yw_login(self, yw_proxy, timeout=180, insecure=False):
        Output.log("Attempting to log in...")
        start_time = time.time()
        # Only the last error message is recorded if timed out.
        err = ""
        while time.time() - start_time < timeout:
            err = yw_proxy.insecure_login() if insecure else yw_proxy.login()
            if not err:
                Output.log("Login succeeded.")
                return ""
            time.sleep(.5)
        Output.log("Failed to login: {}".format(err))
        return "Timeout: " + err

    def check_pg_isready(self, timeout = 5, retries = 10):
        advertise_ip = self.advertise_ip()
        ysql_port = self.configs.saved_data.get("ysql_port")
        path = find_binary_location("pg_isready")
        cmd = [path, "-h", str(advertise_ip), "-p", str(ysql_port)]

        (out, err, retcode) = run_process_with_retries(cmd=cmd, log_cmd=True, retries=retries,
                                                       timeout=timeout)
        Output.log("pg_isready command: {}".format(cmd))
        Output.log("pg_isready output: {}".format(out))
        Output.log("pg_isready error: {}".format(err))
        Output.log("pg_isready return code: {}".format(retcode))
        if retcode:
            return False
        else:
            return True

    # Returns pretty output table.
    def get_status_string(self):

        join_ip = self.configs.saved_data.get("join")
        advertise_ip = self.advertise_ip()
        was_already_setup = self.configs.saved_data.get("cluster_member", False)

        # It is possible that masters have changed, try to find the latest
        # masters before computing the status string.
        master_web_addrs_ip = join_ip
        master_addrs = self.get_current_masters_from_api(join_ip)
        if not master_addrs:
            master_web_addrs_ip = advertise_ip
            master_addrs = self.get_current_masters_from_api(advertise_ip)
        self.configs.saved_data["current_masters"] = master_addrs
        Output.log("Master address list updated, new list: {}".format(master_addrs))

        master_web_addrs = "{}:{}".format(get_url_from_ip(master_web_addrs_ip),
                                          self.configs.saved_data.get("master_webserver_port"))

        ycql_port = self.configs.saved_data.get("ycql_port")
        cql_hostname_param = ""
        cql_port_param = ""
        if advertise_ip != IP_LOCALHOST or ycql_port != DEFAULT_YCQL_PORT:
            cql_hostname_param =  advertise_ip
            cql_port_param = str(ycql_port)
        ycql_username, ycql_password, ycql_keyspace = self.setup_env_init.get_ycql_credentials()

        ysql_hostname_param = "-h {}".format(advertise_ip) if advertise_ip != IP_LOCALHOST else ""
        ysql_port = self.configs.saved_data.get("ysql_port")
        ysql_port_param = "-p {}".format(ysql_port) if ysql_port != DEFAULT_YSQL_PORT else ""
        ysql_username, ysql_password, ysql_db = self.setup_env_init.get_ysql_credentials()

        status_info = []
        status_display_info = dict()
        ret_code = 0
        # Make sure ascii escape characters for color encoding do not count towards char limit.
        if self.get_failed_node_processes():
            title = Output.make_bold(Output.make_red(SCRIPT_NAME))
            extra_len = len(Output.make_bold(Output.make_red("")))
            status = "Stopped"
            ysql_status = "Not Ready"
            status_info = [
                (Output.make_yellow("Status"), status),
                (Output.make_yellow("YSQL Status"), ysql_status),
            ]
            ret_code = 1
        else:
            title = Output.make_bold(Output.make_green(SCRIPT_NAME))
            extra_len = len(Output.make_bold(Output.make_green("")))
            # Check if there is a leader yet.
            # We can use a smaller timeout here instead of the regular 60 secs.
            # In case of starting up via yugabyted, we have already given enough time to
            # the leader election
            # In case of manual start or some other route, we can have a smaller timeout
            status = ""
            if self.configs.temp_data.get("yugabyted_cmd") == "start":
                Output.init_animation("Checking YSQL Status...")
            pg_isready = self.check_pg_isready()
            if self.configs.temp_data.get("yugabyted_cmd") == "start":
                Output.update_animation("", Output.ANIMATION_STOP)
            if was_already_setup:
                if master_addrs:
                    status = "Running."
                    if pg_isready:
                        ysql_status = "Ready"
                        ret_code = 0
                    else:
                        ysql_status = "Not Ready"
                        ret_code = 1

                else:
                    status = "Bootstrapping."
                    ysql_status = "Not Ready"
                    ret_code = 0
            else:
                if self.wait_get_all_masters(timeout=10):
                    status = "Running."
                    if pg_isready:
                        ysql_status = "Ready"
                        ret_code = 0
                    else:
                        ysql_status = "Not Ready"
                        ret_code = 1
                else:
                    status = "Status command timed out as YugabyteDB \"yb-master\" " + \
                                "process is not responding."
                    ysql_status = "Not Ready"
                    ret_code = 1

            enabled_security_features = []
            if self.configs.temp_data.get("yugabyted_cmd") == "status":
                encryption_enabled = self.is_encryption_at_rest_enabled()
                if encryption_enabled is None:
                    Output.log("Error checking status of " +
                        "encryption-at-rest.")
                elif encryption_enabled:
                        enabled_security_features.append("Encryption-at-rest")

            is_secure = self.configs.saved_data.get("secure")
            if is_secure:
                if self.is_leader_master_secure(master_web_addrs):
                    enabled_security_features.append("Encryption-in-transit")
                if self.configs.saved_data.get("ysql_enable_auth") and \
                    self.configs.saved_data.get("use_cassandra_authentication"):
                    enabled_security_features.append("Password Authentication")

            rf = self.configs.temp_data.get("replication_factor")
            if not rf and self.configs.temp_data.get("yugabyted_cmd") == "start":
                Output.init_animation("Verifying data placement constraint " +
                            "on the YugabyteDB Cluster...")
                rf = YBAdminProxy.get_cluster_rf(master_addrs)
                Output.update_animation("Data placement constraint successfully verified")
            else:
                rf = YBAdminProxy.get_cluster_rf(master_addrs)

            status_info = [
                (Output.make_yellow("Status"), status),
                (Output.make_yellow("YSQL Status"), ysql_status),
            ]
            if rf:
                status_info.append((Output.make_yellow("Replication Factor"), rf))

            if enabled_security_features:
                status_info += [
                    (Output.make_yellow("Security Features"), ", ".join(enabled_security_features))
                ]


        ysql_flags = " {} {} -U {} -d {}".format(ysql_hostname_param, ysql_port_param,
                        ysql_username, ysql_db)
        ycql_flags = " {} {} -u {}".format(cql_hostname_param, cql_port_param,
                        ycql_username)
        if ycql_keyspace is not None:
            ycql_flags = ycql_flags + " -k {}".format(ycql_keyspace)
        if self.configs.saved_data.get("secure"):
            ycql_flags = ycql_flags + " --ssl"

        # TODO: Check if YW is disabled. This could be from saving --ui flag, checking PID,
        # or some other method. The first is not preferred because it would deviate from how the
        # other saved data works in that they persist between runs but --ui shoudln't.
        advertise_url_ip = get_url_from_ip(advertise_ip)

        yw_status = "http://{}:{}".format(advertise_url_ip,
                                          self.configs.saved_data.get("webserver_port"))
        yugabyted_ui_status = "http://{}:{}".format(advertise_url_ip,
                                          self.configs.saved_data.get("yugabyted_ui_port"))
        yb_master_status = "http://{}:{}".format(advertise_url_ip,
                                          self.configs.saved_data.get("master_webserver_port"))
        console_desc = "Web console"

        ui_port_available = self.configs.temp_data.get("ui_port_available")
        if self.configs.saved_data.get("ui") and ui_port_available:
            try:
                response = urlopen(Request(yugabyted_ui_status))
                if response.code == 200:
                    console_desc = "YugabyteDB UI"
                    status_info += [ (Output.make_yellow(console_desc), yugabyted_ui_status), ]
                else:
                    status_info += [ (Output.make_yellow(console_desc), yb_master_status), ]
            except HTTPError as http_err:
                Output.log('HTTP error occurred while fetching YugabyteDB UI {}'.format(http_err))
                status_info += [ (Output.make_yellow(console_desc), yb_master_status), ]
            except Exception as err:
                Output.log('Other error occurred while fetching YugabyteDB UI {}'.format(err))
                status_info += [ (Output.make_yellow(console_desc), yb_master_status), ]
        else:
            status_info += [ (Output.make_yellow(console_desc), yb_master_status), ]

        jdbc_string = "jdbc:postgresql://{}:{}/{}?user={}&password=".format(
            advertise_url_ip,
            ysql_port,
            ysql_db,
            ysql_username)
        if self.configs.saved_data.get("secure"):
            jdbc_string += Output.make_cyan("<DB_PASSWORD>")
            status_display_info[jdbc_string] = Output.make_cyan
        else:
            jdbc_string += "yugabyte"

        status_info += [
            (Output.make_yellow("JDBC"), jdbc_string),
            (Output.make_yellow("YSQL"), "bin/ysqlsh{}".format(ysql_flags)),
            (Output.make_yellow("YCQL"), "bin/ycqlsh{}".format(ycql_flags)),
            (Output.make_yellow("Data Dir"), self.configs.saved_data.get("data_dir")),
            (Output.make_yellow("Log Dir"), self.configs.saved_data.get("log_dir")),
            (Output.make_yellow("Universe UUID"), self.configs.saved_data.get("universe_uuid"))
        ]
        max_key_len = 0
        max_value_len = 0
        for k, v in status_info:
            if status_display_info is not None and v in status_display_info.keys():
                func = status_display_info.get(v)
                value_len = len(str(v)) - len(func(""))
            else:
                value_len = len(str(v))

            max_key_len = max(max_key_len, len(str(k)))
            max_value_len = max(max_value_len, value_len)

        div_line = "+" + "-" * (max_key_len + max_value_len + 4) + "+" + "\n"
        status = "\n" + div_line
        status += ("| {:^" + str(max_key_len + max_value_len + 2
            + extra_len) + "} |\n").format(title)
        status += div_line
        for k, v in status_info:
            func = None
            if status_display_info is not None and v in status_display_info.keys():
                func = status_display_info.get(v)

            extra_len = len(Output.make_yellow(""))
            status += ("| {:" + str(max_key_len + extra_len - 7) + "}: ").format(k)
            if func is not None:
                status += ("{:<" + str(max_value_len + 7 + len(func(""))) + "} |\n").format(v)
            else:
                status += ("{:<" + str(max_value_len + 7) + "} |\n"). \
                    format(v if v is not None else "None")

        status += div_line
        return status, ret_code

    # Returns pretty output table
    def get_status_string_common(self, status_info, status_display_info = None):
        title = Output.make_bold(Output.make_green(SCRIPT_NAME))
        extra_len = len(Output.make_bold(Output.make_green("")))

        max_key_len = 0
        max_value_len = 0
        for k, v in status_info:
            max_key_len = max(max_key_len, len(str(k)))
            max_value_len = max(max_value_len, len(str(v)))

        div_line = "+" + "-" * (max_value_len + max_key_len + 5) + "+" + "\n"
        status = "\n" + div_line
        status += ("| {:^" + str(max_value_len + max_key_len + 3
                                 + extra_len) + "} |\n").format(title)
        status += div_line
        for k, v in status_info:
            func = None
            if status_display_info is not None and v in status_display_info.keys():
                func = status_display_info.get(v)
            extra_len = len(Output.make_yellow(""))
            status += ("| {:" + str(max_key_len + extra_len - 1) + "}: ").format(k)
            if func is not None:
                status += ("{:<" + str(max_value_len + 2 + len(func(""))) + "} |\n").format(func(v))
            else:
                status += ("{:<" + str(max_value_len + 2) + "} |\n").format(v)

        status += div_line
        return status


    def start_callhome_thread(self, callhome_event_name, target, use_shutdown_event=False):
        """Start a callhome thread with optional graceful shutdown support.

        Args:
            callhome_event_name: Unique identifier for this callhome thread
            target: Callable function to run in the thread
            use_shutdown_event: If True, create a shutdown event for graceful shutdown

        Returns:
            Thread object that was started
        """
        shutdown_event = None
        if use_shutdown_event:
            shutdown_event = Event()
            self.callhome_shutdown_events[callhome_event_name] = shutdown_event

        callhome_thread = Thread(target=target,
            args=(shutdown_event,) if use_shutdown_event else ())
        callhome_thread.daemon = True
        callhome_thread.start()
        self.callhome_threads[callhome_event_name] = callhome_thread

    def shutdown_callhome_thread(self, callhome_event_name=None, timeout=5):
        """Shutdown a specific callhome thread or all callhome threads gracefully.

        Args:
            callhome_event_name: Name of the thread to shutdown, or None to shutdown all
            timeout: Maximum time to wait for thread completion in seconds

        Returns:
            True if all threads shut down successfully, False otherwise
        """
        if callhome_event_name is None:
            # Shutdown all callhome threads
            success = True
            # Set all shutdown events
            for event in self.callhome_shutdown_events.values():
                event.set()
            # Wait for all threads to complete
            for name, thread in list(self.callhome_threads.items()):
                if thread.is_alive():
                    thread.join(timeout=timeout)
                    if thread.is_alive():
                        success = False
                        Output.log(
                            "Callhome thread '{}' did not shutdown within "
                            "timeout".format(name)
                        )
            # Clear dictionaries
            self.callhome_threads.clear()
            self.callhome_shutdown_events.clear()
            if success:
                Output.log("All callhome threads shut down successfully")
            return success
        else:
            # Shutdown specific thread
            thread = self.callhome_threads.get(callhome_event_name)
            event = self.callhome_shutdown_events.get(callhome_event_name)

            if event:
                event.set()

            if thread:
                if thread.is_alive():
                    thread.join(timeout=timeout)
                    if thread.is_alive():
                        Output.log(
                            "Callhome thread '{}' did not shutdown within "
                            "timeout".format(callhome_event_name)
                        )
                        return False
                # Remove from dictionaries
                self.callhome_threads.pop(callhome_event_name, None)
                self.callhome_shutdown_events.pop(callhome_event_name, None)
                Output.log("Callhome thread '{}' shut down successfully".format(
                    callhome_event_name))
                return True
            return False

    def _get_cluster_data(self, target_ip=None):
        """Fetch cluster data from UI API using generic framework.

        Args:
            target_ip: Optional IP address to fetch from (default: local cluster)

        Returns:
            Tuple of (cluster data dictionary, response_code) or (None, None) if request fails
        """
        return self._fetch_api_endpoint('/api/cluster', 'cluster data', target_ip=target_ip)

    def _get_alerts_info(self):
        """Fetch alerts from UI API and extract all info strings.

        Returns:
            List of alert info strings, or empty list if API call fails or no alerts
        """
        alerts_data, _ = self._fetch_api_endpoint('/api/alerts', 'alerts')
        alert_info_strings = []

        if alerts_data and 'data' in alerts_data:
            for alert in alerts_data['data']:
                if 'info' in alert:
                    alert_info_strings.append(alert['info'])

        return alert_info_strings

    def _get_ysql_databases(self):
        """Fetch YSQL tables and extract unique keyspaces/databases.

        Returns:
            List of unique database/keyspace names, or empty list if API call fails
        """
        tables_data, _ = self._fetch_api_endpoint(
            '/api/tables', 'YSQL tables',
            query_params={'api': 'YSQL'}
        )
        databases = []

        if tables_data and 'tables' in tables_data:
            for table in tables_data['tables']:
                if 'keyspace' in table:
                    keyspace = table['keyspace']
                    if keyspace and keyspace not in databases:
                        databases.append(keyspace)

        return databases

    def _get_index_backfill_activities(self):
        """Fetch INDEX_BACKFILL activities for both COMPLETED and IN_PROGRESS.

        Returns structured dictionary with in_progress and completed lists,
        with PII fields filtered out.

        Returns:
            Dictionary with structure: {"in_progress": [...], "completed": [...]}
        """
        activities = {
            'in_progress': [],
            'completed': []
        }

        # Fetch COMPLETED activities
        completed_data, _ = self._fetch_api_endpoint(
            '/api/activities', 'INDEX_BACKFILL completed activities',
            query_params={
                'activities': 'INDEX_BACKFILL',
                'status': 'COMPLETED'
            }
        )

        if completed_data and 'data' in completed_data:
            for activity in completed_data['data']:
                if 'data' in activity:
                    activity_data = activity['data']
                    # Filter PII field: IndexName (only PII field in COMPLETED)
                    filtered_data = {
                        k: v for k, v in activity_data.items()
                        if k != 'IndexName'
                    }
                    if filtered_data:
                        activities['completed'].append(filtered_data)

        # Fetch IN_PROGRESS activities for each YSQL database
        databases = self._get_ysql_databases()
        for database in databases:
            in_progress_data, _ = self._fetch_api_endpoint(
                '/api/activities', 'INDEX_BACKFILL in_progress activities',
                query_params={
                    'activities': 'INDEX_BACKFILL',
                    'status': 'IN_PROGRESS',
                    'database': database
                }
            )

            if in_progress_data and 'data' in in_progress_data:
                for activity in in_progress_data['data']:
                    if 'data' in activity:
                        activity_data = activity['data']
                        # Filter PII fields: DBName, IndexName, Command
                        filtered_data = {
                            k: v for k, v in activity_data.items()
                            if k not in ['DBName', 'IndexName', 'Command']
                        }
                        if filtered_data:
                            activities['in_progress'].append(filtered_data)

        return activities

    def _fetch_api_endpoint(self, endpoint_path, description="", target_ip=None,
                            query_params=None):
        """Generic method to fetch data from UI API endpoints.

        For read replica nodes, API calls are made to the leader master node
        (join IP) since read replicas don't have their own master.

        Args:
            endpoint_path: API endpoint path (e.g., '/api/nodes')
            description: Description for logging purposes
            target_ip: Optional IP address to fetch from (default: local cluster)
            query_params: Optional dictionary of query parameters to append to URL

        Returns:
            Tuple of (parsed JSON data, response_code) or (None, None) if request fails
        """
        try:
            if target_ip is None:
                # For read replica nodes, use the join IP (leader master) for API calls
                if self.configs.saved_data.get("read_replica"):
                    target_ip = self.configs.saved_data.get("join")
                else:
                    target_ip = self.advertise_ip()

            target_url_ip = get_url_from_ip(target_ip)
            ui_port = self.configs.saved_data.get("yugabyted_ui_port")
            api_url = "http://{}:{}{}".format(target_url_ip, ui_port, endpoint_path)

            # Append query parameters if provided
            if query_params:
                query_string = urlencode(query_params)
                api_url = "{}?{}".format(api_url, query_string)

            response = urlopen(Request(api_url))
            if response.code == 200:
                data = json.loads(response.read().decode('utf-8'))
                return (data, response.code)
            else:
                return (None, response.code)
        except Exception as e:
            log_msg = "Failed to fetch {} from API: {}".format(
                description if description else endpoint_path, str(e))
            Output.log(log_msg)

        return (None, None)

    def _format_time_taken_ms(self, start_time, end_time):
        """Format time difference in milliseconds as string."""
        time_diff_ms = int((end_time - start_time) * 1000)
        return "{} ms".format(time_diff_ms)

    def get_common_callhome_payload(self, command: str, cluster_data=None, response_code=None):
        callhome_payload = CallhomePayload(
            callhome_type=CALLHOME_TYPE,
            command=command,
            timestamp=datetime.now().isoformat(),
            node_uuid=self.configs.saved_data.get("node_uuid", ""),
            cluster_uuid="",
            universe_uuid="",
            is_ui_up=False,
            is_read_replica=self.configs.saved_data.get("read_replica", False),
            payload=dict()
        )

        # Fetch cluster data if not provided
        if cluster_data is None or response_code is None:
            cluster_data, response_code = self._get_cluster_data()

        # Extract cluster and universe UUID from cluster data
        if cluster_data and 'data' in cluster_data and 'info' in cluster_data['data']:
            callhome_payload.cluster_uuid = cluster_data['data']['info'].get('cluster_uuid', '')
            callhome_payload.universe_uuid = cluster_data['data']['info'].get('universe_uuid', '')

        # Set UI status based on response code from cluster data API call
        callhome_payload.is_ui_up = (response_code == 200)

        callhome_payload.timestamp = datetime.now().isoformat()

        return callhome_payload

    def _send_backup_restore_callhome(self, event_type, task_id=None):
        """Send callhome event for backup or restore command.

        Args:
            event_type: CALLHOME_EVENT_BACKUP or CALLHOME_EVENT_RESTORE
            task_id: Task ID for the backup/restore operation (default None)
        """
        callhome_payload = self.get_common_callhome_payload(event_type)

        # Determine ns_type from config
        ns_type = ""
        if self.configs.temp_data.get("database_name_backup_restore"):
            ns_type = "ysql"
        elif self.configs.temp_data.get("keyspace_name_backup_restore"):
            ns_type = "ycql"

        payload = {
            'cloud_type': self.configs.temp_data.get("ybc_cloud_type"),
            'ns_type': ns_type
        }

        # Add task_id if provided
        if task_id:
            payload['task_id'] = task_id

        # Add is_pitr_restore only for restore events
        if event_type == CALLHOME_EVENT_RESTORE:
            payload['is_pitr_restore'] = bool(
                self.configs.temp_data.get("restore_time"))

        callhome_payload.payload = payload
        self.diagnostics.callhome(callhome_payload)

    def _send_xcluster_callhome(self, event_type,
                                success=True, failure_reason=None,
                                database_count=None,
                                target_address=None, automatic_mode=None,
                                is_target_available=None, time_taken=None):
        """Send callhome event for xcluster operations.

        Args:
            event_type: CALLHOME_EVENT_XCLUSTER_* constant
            success: Whether operation succeeded (default True)
            failure_reason: Failure reason if not successful (default None)
            database_count: Number of databases (default None)
            target_address: Target cluster address (default None)
            automatic_mode: Automatic mode flag for create_checkpoint (default None)
            is_target_available: Target availability for delete_replication (default None)
            time_taken: Time taken for the operation in "XXXX ms" format (default None)
        """
        callhome_payload = self.get_common_callhome_payload(event_type)

        payload = {
            'success': success
        }
        if time_taken:
            payload['time_taken'] = time_taken

        # Add database_count if provided
        if database_count is not None:
            payload['database_count'] = database_count

        # Add optional fields based on parameters
        if automatic_mode is not None:
            payload['automatic_mode'] = bool(automatic_mode)

        if is_target_available is not None:
            payload['target_available'] = is_target_available

        # Add failure_reason if provided
        if not success and failure_reason:
            payload['failure_reason'] = failure_reason

        # Get target cluster info if target_address provided
        if target_address:
            cluster_data, _ = self._get_cluster_data(target_ip=target_address)
            if cluster_data and 'data' in cluster_data and \
               'info' in cluster_data['data']:
                target_cluster_uuid = cluster_data['data']['info'].get(
                    'cluster_uuid', '')
                target_universe_uuid = cluster_data['data']['info'].get(
                    'universe_uuid', '')
                if target_cluster_uuid:
                    payload['target_cluster_uuid'] = target_cluster_uuid
                if target_universe_uuid:
                    payload['target_universe_uuid'] = target_universe_uuid

        callhome_payload.payload = payload
        self.diagnostics.callhome(callhome_payload)

    def _send_pitr_enable_callhome(self, pitr_yb_api_type, retention_period,
                                    success, time_taken):
        """Send callhome event for PITR enable."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_PITR_ENABLE
        )
        retention_period_days = None
        if retention_period:
            retention_period_days = retention_period // 1440
        callhome_payload.payload = {
            'pitr_type': pitr_yb_api_type,
            'retention_period_days': retention_period_days,
            'success': success,
            'time_taken': time_taken
        }
        self.diagnostics.callhome(callhome_payload)

    def _send_pitr_disable_callhome(self, pitr_yb_api_type, success,
                                     time_taken):
        """Send callhome event for PITR disable."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_PITR_DISABLE
        )
        callhome_payload.payload = {
            'pitr_type': pitr_yb_api_type,
            'success': success,
            'time_taken': time_taken
        }
        self.diagnostics.callhome(callhome_payload)

    def _send_demo_connect_callhome(self, demo_db, time_taken):
        """Send callhome event for demo connect."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_DEMO_CONNECT
        )
        callhome_payload.payload = {
            'demo_db': demo_db,
            'time_taken': time_taken
        }
        self.diagnostics.callhome(callhome_payload)

    def _send_upgrade_ysql_catalog_callhome(self, success, time_taken):
        """Send callhome event for upgrade ysql catalog."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_UPGRADE_YSQL_CATALOG
        )
        callhome_payload.payload = {
            'success': success,
            'time_taken': time_taken
        }
        self.diagnostics.callhome(callhome_payload)

    def _send_upgrade_finalize_new_version_callhome(self, target_version,
                                                     node_count, timeout_ms,
                                                     success, failure_reason=None,
                                                     time_taken=None):
        """Send callhome event for upgrade finalize new version."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_UPGRADE_FINALIZE_NEW_VERSION
        )
        payload = {
            'target_version': target_version,
            'success': success
        }
        if not success and failure_reason:
            payload['failure_reason'] = failure_reason
        if time_taken:
            payload['time_taken'] = time_taken
        callhome_payload.payload = payload
        self.diagnostics.callhome(callhome_payload)

    def _send_upgrade_check_version_compatibility_callhome(self, compatible, time_taken):
        """Send callhome event for upgrade check version compatibility."""
        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_UPGRADE_CHECK_VERSION_COMPATIBILITY
        )
        callhome_payload.payload = {
            'compatible': compatible,
            'time_taken': time_taken
        }
        self.diagnostics.callhome(callhome_payload)

    def _get_configuration_data(self):
        """Get configuration data for start command callhome.

        Returns:
            Dictionary containing yugabyted flags and filtered server flags
        """
        # Get common flags
        common_flags = self.get_common_flags()

        # Get full master command (includes common flags, hardcoded flags,
        # user flags, flagfile flags, auto-added flags)
        master_cmd = self.get_master_cmd(common_flags)

        # Get full tserver command (includes common flags, hardcoded flags,
        # user flags, flagfile flags, auto-added flags)
        tserver_cmd = self.get_tserver_cmd(common_flags)

        # Extract flags from command lists
        master_flags_list = self._extract_flags_from_cmd(master_cmd)
        tserver_flags_list = self._extract_flags_from_cmd(tserver_cmd)

        # Load allowlist once for both master and tserver filtering
        allowlist = self._load_gflag_allowlist()

        # Filter flags based on allowlist
        filtered_master_flags = self._filter_flags_by_allowlist(
            master_flags_list, allowlist, "master")
        filtered_tserver_flags = self._filter_flags_by_allowlist(
            tserver_flags_list, allowlist, "tserver")

        config_data = {
            'yugabyted_flags': {
                'tls': self.configs.saved_data.get("secure"),
                'password_authentication': (
                    self.configs.saved_data.get("ysql_enable_auth") and
                self.configs.saved_data.get("use_cassandra_authentication")
                ),
                'clockbound': self.configs.temp_data.get("enhance_time_sync_via_clockbound"),
                'backup_daemon': self.configs.saved_data.get("backup_daemon")
            },
            'master_flags': filtered_master_flags,
            'tserver_flags': filtered_tserver_flags
        }
        return config_data

    def _get_cluster_topology_data(self, cluster_data=None):
        """Get cluster topology data for start command callhome.

        Args:
            cluster_data: Optional cluster data dictionary. If not provided, will be fetched.

        Returns:
            Dictionary containing replication factor, fault tolerance, and cluster size
        """
        topology_data = {
            'replication_factor': '',
            'fault_tolerance': '',
            'cluster_size': 0
        }

        # Fetch cluster data if not provided
        if cluster_data is None:
            cluster_data, _ = self._get_cluster_data()

        if (cluster_data and 'data' in cluster_data and
                'spec' in cluster_data['data'] and
                'cluster_info' in cluster_data['data']['spec']):
            cluster_info = cluster_data['data']['spec']['cluster_info']
            topology_data['replication_factor'] = cluster_info.get('replication_factor', '')
            topology_data['fault_tolerance'] = cluster_info.get('fault_tolerance', '')
            topology_data['cluster_size'] = cluster_info.get('num_nodes', 0)
        else:
            Output.log("Cluster data not available for deployment topology, using defaults")

        return topology_data

    def start_cmd_callhome(self):
        # Fetch cluster data once to avoid duplicate API calls
        Output.log("Gathering {} callhome payload".format(CALLHOME_EVENT_START))
        cluster_data, response_code = self._get_cluster_data()

        callhome_payload = self.get_common_callhome_payload(
            CALLHOME_EVENT_START,
            cluster_data=cluster_data,
            response_code=response_code
        )

        # Determine if this is the first node or an added node
        join_ip = self.configs.saved_data.get("join")

        # Do we need following feilds for heartbeat_status events also?
        is_first_node = not bool(join_ip)
        is_added_node = bool(join_ip)

        # Build categorized payload structure
        payload = {
            'configuration': self._get_configuration_data(),
            'cluster_topology': self._get_cluster_topology_data(
                cluster_data=cluster_data
            ),
            'is_first_node': is_first_node,
            'is_added_node': is_added_node
        }

        # List of details pending to be sent
        # Time to first query: Time taken to run the first SQL Query after cluster started

        callhome_payload.payload = payload
        callhome_payload.timestamp = datetime.now().isoformat()

        Output.log("Finished gathering '{}' callhome payload, sending to diagnostics".format(
                CALLHOME_EVENT_START
            ))
        self.diagnostics.callhome(callhome_payload)


    def _get_node_performance_metrics(self, nodes_data, current_node_ip):
        """Get performance metrics for current node.

        Args:
            nodes_data: API response data containing node information
            current_node_ip: IP address of the current node

        Returns:
            Dictionary containing CPU, disk, and uptime metrics
        """
        performance_metrics = {}

        if not nodes_data or 'data' not in nodes_data:
            return performance_metrics

        for node in nodes_data['data']:
            if node.get('host') == current_node_ip:
                metrics = node.get('metrics', {})
                performance_metrics['cpu_utilization'] = metrics.get('cpu_usage', 0)
                performance_metrics['disk_size_used_gb'] = metrics.get('disk_size_used_gb', 0)
                performance_metrics['disk_size_gb'] = metrics.get('disk_size_gb', 0)
                performance_metrics['server_uptime_seconds'] = metrics.get('uptime_seconds', 0)
                break

        return performance_metrics

    def _get_node_connection_metrics(self, nodes_data, current_node_ip):
        """Get connection metrics for current node.

        Args:
            nodes_data: API response data containing node information
            current_node_ip: IP address of the current node

        Returns:
            Dictionary containing YSQL and YCQL connection metrics
        """
        connection_metrics = {}

        if not nodes_data or 'data' not in nodes_data:
            return connection_metrics

        for node in nodes_data['data']:
            if node.get('host') == current_node_ip:
                metrics = node.get('metrics', {})
                connections = metrics.get('active_connections', {})
                connection_metrics['ysql_active_connections'] = connections.get('ysql', 0)
                connection_metrics['ycql_active_connections'] = connections.get('ycql', 0)
                connection_metrics['ysql_in_use'] = connections.get('ysql', 0) > 0
                connection_metrics['ycql_in_use'] = connections.get('ycql', 0) > 0
                break

        return connection_metrics

    def _get_node_operation_metrics(self, nodes_data, current_node_ip):
        """Get operation metrics for current node.

        Args:
            nodes_data: API response data containing node information
            current_node_ip: IP address of the current node

        Returns:
            Dictionary containing read and write operations per second
        """
        operation_metrics = {}

        if not nodes_data or 'data' not in nodes_data:
            return operation_metrics

        for node in nodes_data['data']:
            if node.get('host') == current_node_ip:
                metrics = node.get('metrics', {})
                operation_metrics['read_ops_per_sec'] = metrics.get('read_ops_per_sec', 0)
                operation_metrics['write_ops_per_sec'] = metrics.get('write_ops_per_sec', 0)
                break

        return operation_metrics

    def _get_node_tablet_metrics(self, nodes_data, current_node_ip):
        """Get tablet metrics for current node.

        Args:
            nodes_data: API response data containing node information
            current_node_ip: IP address of the current node

        Returns:
            Dictionary containing user and system tablet counts
        """
        tablet_metrics = {}

        if not nodes_data or 'data' not in nodes_data:
            return tablet_metrics

        for node in nodes_data['data']:
            if node.get('host') == current_node_ip:
                metrics = node.get('metrics', {})
                tablet_metrics['user_tablets'] = metrics.get('user_tablets_total', 0)
                tablet_metrics['user_tablet_leaders'] = metrics.get('user_tablets_leaders', 0)
                tablet_metrics['system_tablets'] = metrics.get('system_tablets_total', 0)
                tablet_metrics['system_tablet_leaders'] = metrics.get('system_tablets_leaders', 0)
                break

        return tablet_metrics

    def _get_cluster_node_info(self, nodes_data):
        """Get cluster-wide node count information.

        Args:
            nodes_data: API response data containing node information

        Returns:
            Dictionary containing total and active node counts
        """
        node_info = {
            'total_nodes_count': 0,
            'active_nodes_count': 0
        }

        if not nodes_data or 'data' not in nodes_data:
            return node_info

        node_info['total_nodes_count'] = len(nodes_data['data'])
        node_info['active_nodes_count'] = sum(
            1 for node in nodes_data['data'] if node.get('is_node_up', False))

        return node_info

    def _get_cluster_health_metrics(self):
        """Get cluster-wide health metrics.

        Returns:
            Dictionary containing load balancer status and tablet health
        """
        health_metrics = {}

        # Fetch load balancer status
        lb_data, _ = self._fetch_api_endpoint('/api/is_load_balancer_idle', 'load balancer status')
        if lb_data:
            health_metrics['is_load_balancer_idle'] = lb_data.get('is_idle', None)

        # Fetch tablet health from health check API
        health_data, _ = self._fetch_api_endpoint('/api/health-check', 'health check')
        if health_data and 'data' in health_data:
            health_metrics['under_replicated_tablets'] = (
                health_data['data'].get('under_replicated_tablets', 0))
            health_metrics['unavailable_tablets'] = (
                health_data['data'].get('leaderless_tablets', 0))

        return health_metrics

    def _get_cluster_database_metrics(self):
        """Get cluster-wide database metrics.

        Returns:
            Dictionary containing table, index, and migration counts
        """
        database_metrics = {}

        # Fetch migrations count
        migrations_data, _ = self._fetch_api_endpoint('/api/migrations', 'migrations')
        if (migrations_data and 'data' in migrations_data and
                isinstance(migrations_data['data'], list)):
            database_metrics['migrations_count'] = len(migrations_data['data'])

        # Fetch table and index counts using API
        # Fetch YSQL tables and indexes
        ysql_data, _ = self._fetch_api_endpoint(
            '/api/tables', 'YSQL tables',
            query_params={'api': 'YSQL'}
        )
        ysql_tables_count = 0
        ysql_indexes_count = 0
        if ysql_data and 'tables' in ysql_data:
            ysql_tables_count = len(ysql_data['tables'])
        if ysql_data and 'indexes' in ysql_data:
            ysql_indexes_count = len(ysql_data['indexes'])

        # Fetch YCQL tables and indexes
        ycql_data, _ = self._fetch_api_endpoint(
            '/api/tables', 'YCQL tables',
            query_params={'api': 'YCQL'}
        )
        ycql_tables_count = 0
        ycql_indexes_count = 0
        if ycql_data and 'tables' in ycql_data:
            ycql_tables_count = len(ycql_data['tables'])
        if ycql_data and 'indexes' in ycql_data:
            ycql_indexes_count = len(ycql_data['indexes'])

        database_metrics['user_tables_count'] = {
            'YSQL': ysql_tables_count,
            'YCQL': ycql_tables_count
        }
        database_metrics['user_indexes_count'] = {
            'YSQL': ysql_indexes_count,
            'YCQL': ycql_indexes_count
        }

        return database_metrics

    def _load_gflag_allowlist(self):
        """Load gflag allowlist from file.

        Returns:
            Set of allowed flag names (without -- prefix) or empty set
            if file not found
        """
        # Build dir_candidates list for allowlist file directories
        # (same pattern as search_file_in_paths)
        dir_candidates = [
            # Release environment
            YUGABYTE_DIR,
            # Dev environment
            os.path.join(YUGABYTE_DIR, "src", "yb")
        ]

        # Use search_file_in_paths pattern to find the file
        allowlist_file = None
        for candidate_dir in dir_candidates:
            candidate_path = os.path.join(
                candidate_dir, "gflag_allowlist.txt")
            if os.path.exists(candidate_path):
                Output.log("Found directory {} for file {}".format(
                    candidate_dir, "gflag_allowlist.txt"))
                allowlist_file = candidate_path
                break

        if not allowlist_file:
            Output.log(
                "gflag_allowlist.txt not found, master and tserver flags "
                "will not be sent", logging.WARN)
            return set()

        allowed_flags = set()
        try:
            with open(allowlist_file, 'r') as f:
                for line in f:
                    line = line.strip()
                    # Skip commented lines and empty lines
                    if not line or line.startswith('#'):
                        continue
                    # Flag names in file do NOT have -- prefix, use as-is
                    allowed_flags.add(line)
        except Exception as e:
            Output.log("Failed to read gflag allowlist file: {}".format(str(e)))
            return set()

        return allowed_flags

    def _extract_flags_from_cmd(self, cmd_list):
        """Extract flags from command list returned by get_master_cmd or
        get_tserver_cmd.

        Args:
            cmd_list: List returned from get_master_cmd() or
                     get_tserver_cmd() (first element is binary path)

        Returns:
            List of flag strings (e.g., ["--flag1=value1", "--flag2=value2"])
        """
        if not cmd_list or len(cmd_list) < 2:
            return []

        # Skip first element (binary path) and return rest (flags)
        return cmd_list[1:]

    def _filter_flags_by_allowlist(
            self, flags_list, allowlist, flag_type="master"):
        """Filter flags based on allowlist.

        Args:
            flags_list: List of flag strings
                       (e.g., ["--flag1=value1", "--flag2=value2"])
            allowlist: Set of allowed flag names (without -- prefix)
            flag_type: "master" or "tserver" for logging

        Returns:
            Filtered list of flag strings (empty list if allowlist is empty)
        """
        # If allowlist is empty (file not found), return empty list
        if not allowlist:
            return []

        filtered_list = []

        for flag in flags_list:
            try:
                # Extract flag name (remove -- prefix, handle complex flags with =)
                # Handle different flag formats:
                # --flag_name=value -> flag_name
                # --flag_name={value} -> flag_name
                # --flag_name -> flag_name

                if flag.startswith('--'):
                    flag_name = flag[2:]  # Remove '--' prefix
                    # Split on '=' to get flag name (first part)
                    if '=' in flag_name:
                        flag_name = flag_name.split('=', 1)[0]
                    # Check if flag name is in allowlist
                    if flag_name in allowlist:
                        filtered_list.append(flag)
            except Exception as e:
                # Log error for that flag, continue with others
                Output.log("Error parsing flag '{}': {}".format(flag, str(e)))
                continue

        # Log summary
        Output.log(
            "Filtered {} {} flags: {} allowed, {} filtered out".format(
                len(flags_list), flag_type, len(filtered_list),
                len(flags_list) - len(filtered_list)))

        return filtered_list

    def _get_backup_restore_info(self):
        """Fetch backup and restore information from API endpoints.

        Returns:
            Dictionary with 'backup' and 'restore' arrays containing
            start_time, status, time_taken, bytes_transferred, and task_id
            for each entry.
        """
        backup_restore_info = {
            'backup': [],
            'restore': []
        }

        # Fetch backup data
        backup_data, _ = self._fetch_api_endpoint('/api/backup', 'backup')
        if backup_data and 'backup' in backup_data:
            for entry in backup_data['backup']:
                backup_info = {
                    'start_time': entry.get('task_start_time', ''),
                    'status': entry.get('task_status', ''),
                    'time_taken': entry.get('time_taken', ''),
                    'bytes_transferred': entry.get('bytes_transferred', ''),
                    'task_id': entry.get('ybc_task_id', '')
                }
                backup_restore_info['backup'].append(backup_info)

        # Fetch restore data
        restore_data, _ = self._fetch_api_endpoint('/api/restore', 'restore')
        if restore_data and 'restore' in restore_data:
            for entry in restore_data['restore']:
                restore_info = {
                    'start_time': entry.get('task_start_time', ''),
                    'status': entry.get('task_status', ''),
                    'time_taken': entry.get('time_taken', ''),
                    'bytes_transferred': entry.get('bytes_transferred', ''),
                    'task_id': entry.get('ybc_task_id', '')
                }
                backup_restore_info['restore'].append(restore_info)

        return backup_restore_info

    def get_heartbeat_status_payload(self):
        """Fetch node metrics for current node and cluster info from the UI API.

        Returns categorized payload with node-specific and cluster-wide metrics.
        """
        # Fetch nodes data once from API
        nodes_data, _ = self._fetch_api_endpoint('/api/nodes', 'node metrics')
        current_node_ip = self.advertise_ip()

        # Log if current node not found
        if nodes_data and 'data' in nodes_data:
            node_found = any(node.get('host') == current_node_ip for node in nodes_data['data'])
            if not node_found:
                Output.log("Current node {} not found in API response".format(current_node_ip))

        # Build categorized payload structure
        payload = {
            'node_metrics': {
                'performance': self._get_node_performance_metrics(nodes_data, current_node_ip),
                'connections': self._get_node_connection_metrics(nodes_data, current_node_ip),
                'operations': self._get_node_operation_metrics(nodes_data, current_node_ip),
                'tablets': self._get_node_tablet_metrics(nodes_data, current_node_ip)
            },
            'cluster_metrics': {
                'nodes': self._get_cluster_node_info(nodes_data),
                'health': self._get_cluster_health_metrics(),
                'database': self._get_cluster_database_metrics()
            },
            'alerts': self._get_alerts_info(),
            'activities': {
                'index_backfill': self._get_index_backfill_activities()
            },
            'backup_restore': self._get_backup_restore_info()
        }

        return payload

    # Callhome loop. Sends data every minute for the first hour, then every hour after.
    def heartbeat_status_callhome_loop(self, shutdown_event=None):
        num_times_called = 0
        initial_interval = 900 # For the first hour, send data every 15 minutes
        final_interval = 3600 # After the first hour, send data every hour

        # Get shutdown event if not provided (for backward compatibility)
        if shutdown_event is None:
            shutdown_event = self.callhome_shutdown_events.get(
                CALLHOME_EVENT_HEARTBEAT_STATUS
            )

        while shutdown_event is None or not shutdown_event.is_set():
            Output.log("Gathering '{}' callhome payload".format(
                CALLHOME_EVENT_HEARTBEAT_STATUS
            ))

            callhome_payload = self.get_common_callhome_payload(
                CALLHOME_EVENT_HEARTBEAT_STATUS
            )

            # Get heartbeat status payload (categorized structure)
            payload = self.get_heartbeat_status_payload()

            callhome_payload.payload = payload
            callhome_payload.timestamp = datetime.now().isoformat()

            Output.log("Finished gathering '{}' callhome payload, sending to diagnostics".format(
                CALLHOME_EVENT_HEARTBEAT_STATUS
            ))
            self.diagnostics.callhome(callhome_payload)
            num_times_called += 1

            # Use shutdown_event.wait() instead of time.sleep() for responsive shutdown
            timeout = initial_interval if num_times_called < 4 else final_interval
            if shutdown_event:
                shutdown_event.wait(timeout=timeout)
                if shutdown_event.is_set():
                    break
            else:
                time.sleep(timeout)

    # update the master address list every min for the first half hour, then every hour after.
    def update_masters_list_loop(self):
        num_times_cmd_executed = 0
        initial_interval = 60
        final_interval = 3600
        try:
            while True:
                self.update_master_list_on_background_thread()
                num_times_cmd_executed += 1
                time.sleep(initial_interval
                        if num_times_cmd_executed < 30 else final_interval)
        except Exception as e:
                Output.log("update_masters_list: " + str(e))
                pass

    def update_master_list_on_background_thread(self):

        # global _global_current_list_of_masters
        current_master_list = self.configs.saved_data.get("current_masters").split(",")
        Output.log("thread-uml: current masters {}".format(current_master_list))

        # thread for updating the master address list in the background
        # First attempt for /masters using the --join flag
        masters_csv = self.get_current_masters_from_api(self.configs.saved_data.get("join"))
        if not masters_csv:
            # second attempt for /masters using the --advertise_address flag
            masters_csv = self.get_current_masters_from_api(self.advertise_ip())
            if not masters_csv:
                Output.log("thread-uml: Unable to query for all masters list, " +
                    "keeping masters list: {}".format(current_master_list))
                return

        full_master_list = masters_csv.split(",")
        # check if masters have changed since last refresh, if not return
        if_any_new_masters = set(full_master_list).difference(current_master_list)
        if if_any_new_masters:
            self.configs.saved_data["current_masters"] = masters_csv
            Output.log("thread-uml: master list updated, old list: {} ".format(
                current_master_list) + "new list: {}".format(full_master_list))

            tserver_ip_addr = self.advertise_ip()
            tserver_ip_port = self.configs.saved_data.get("tserver_rpc_port")
            if not YBTableServerCLIProxy.update_gflag_in_tserver(tserver_ip_addr,
                tserver_ip_port, "tserver_master_addrs", masters_csv, True):
                Output.log("Failed to update the master list {} " +
                            "on the tserver node.".format(masters_csv))

            # Persist the config if masters have changed
            self.configs.save_configs()

    def get_created_on_timestamp_loop(self):
        num_times_cmd_executed = 0
        initial_interval = 60
        fallback_time = int(time.time())
        try:
            while True:
                self.get_created_on_timestamp()
                if self.configs.saved_data.get("created_on") > 0:
                    break
                num_times_cmd_executed += 1
                # Stop trying to get created_on timestamp after 30 attempts
                if num_times_cmd_executed > 30:
                    self.configs.saved_data["created_on"] = fallback_time
                    self.configs.save_configs()
                    break
                time.sleep(initial_interval)
        except Exception as e:
                Output.log("get_created_on_timestamp: " + str(e))
                pass

    def get_created_on_timestamp(self, timeout = 30):
        join_ip = self.configs.saved_data.get("join")
        if self.configs.saved_data.get("created_on") == 0 and join_ip:
            # Attempt to get created_on date from yugabyted_ui
            yugabyted_ui_addr = "{}:{}".format(
                                get_url_from_ip(join_ip),
                                self.configs.saved_data.get("yugabyted_ui_port"))
            try:
                createdOnAPI = "http://{}/api/created_on".format(yugabyted_ui_addr)
                Output.log("Trying to get cluster creation date from {}".format(createdOnAPI) +
                            " (Timeout={})".format(timeout))
                response = urlopen(Request(createdOnAPI), timeout=timeout)
                responseBody = response.read().decode('utf-8')
                Output.log("Got response from {}: {}".format(createdOnAPI, responseBody))
                self.configs.saved_data["created_on"] = int(responseBody)
                self.configs.save_configs()
            except HTTPError as http_err:
                Output.log('HTTP error occurred while fetching creation timestamp: {}', http_err)
            except URLError as url_err:
                Output.log('URL error occurred while fetching creation timestamp: {}', url_err)
            except Exception as err:
                Output.log('Other error occurred while fetching creation timestamp: {}', err)

    # Calls func after receiving certain exit signals.
    def set_signals(self, func):
        for sig in EXIT_SIGNALS:
            signal(sig, func)

    # Returns true if Java is installed. While Java 8+ is necessary for YW, we don't explicitly
    # check for it so that future version format changes (e.g. Java 1.4 to Java 5) won't give
    # wrong error messages.
    def java_installed(self):
        try:
            cmd = ["java", "-version"]
            java_info = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            return java_info is not None
        except (OSError, subprocess.CalledProcessError) as e:
            Output.log("Failed to find java: {}".format(e), logging.ERROR)
            return False

    # Check whether the hostnames or IP provided is in correct format.
    def validate_hostname_ip(self, ip):
        # Validate any IP address
        try:
            _ = ipaddress.ip_address(ip)
            return True
        except ValueError:
            pass # Attempt dns and hostname validation

        # Regex expression for validating DNS names
        dns_regex = r"^(?=.{1,255}$)(?:(?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.?)+[a-zA-Z]{2,}$"

        # Regex expression for validating hostname
        hostname_regex = "^[a-zA-Z0-9.-]+$"

        dns_pattern = re.compile(dns_regex)
        hostname_pattern = re.compile(hostname_regex)

        # Checking if it is a valid FQDN or hostname
        if (re.search(dns_pattern, ip) or
                re.search(hostname_pattern, ip)):
            self.configs.saved_data["dns_enabled"] = True
            return True

        return False

    def get_replication_groups_for_xcluster(self):
        current_masters = self.configs.saved_data.get("current_masters")

        outbound_replication_groups = YBAdminProxy.get_source_xcluster_replication_ids(
            current_masters)
        if outbound_replication_groups == None:
            Output.log_error_and_exit(Output.make_red("Error") + ": Couldn't retrieve Outbound " +
                                      "Replication Groups.")

        Output.log("Found Outbound Replication Groups: {}".format(outbound_replication_groups))

        return outbound_replication_groups

    def does_replication_id_exists(self, replication_id):
        replication_groups = self.get_replication_groups_for_xcluster()
        return replication_id in replication_groups

    def get_databases_data_for_xcluster(self, group_name = None):
        current_masters = self.configs.saved_data.get("current_masters")

        # response format:
        # [{
        #   "replication_id": <replication_id>,
        #   "databases": [{
        #         "name": <db_name>,
        #         "id"  : <db_id>}]
        #   },
        #  {
        #   "replication_id": <replication_id>,
        #   "databases": [{
        #         "name": <db_name>,
        #         "id"  : <db_id>}]
        #   },.....]
        response = list()

        if group_name:
            response.append(self.get_replication_group_info(group_name, current_masters))
        else:
            replication_groups = self.get_replication_groups_for_xcluster()

            for replication_group in replication_groups:
                response.append(self.get_replication_group_info(replication_group, current_masters))

        Output.log("Replication group infos: {}".format(response))
        return response

    def get_replication_group_info(self, group_name, current_masters):
        databases_data = YBAdminProxy.get_source_xcluster_databases(current_masters, group_name)

        rep_group_info = dict()
        rep_group_info["replication_id"] = group_name
        rep_group_info["databases"] = list()
        for db_name, db_id in databases_data.items():
            rep_group_info["databases"].append({"name": db_name,
                                            "id": db_id})

        return rep_group_info

    def does_database_exist_in_replication(self, database, group_name):
        databases_data = self.get_databases_data_for_xcluster(group_name=group_name)

        rep_group_info = databases_data[0]

        databases = [database_data.get("name") for database_data in rep_group_info["databases"]]
        return database in databases

    def get_xcluster_info(self, masterIP):
        if not masterIP:
            Output.log("Empty masterIP passed to get_xcluster_info")
            return []
        master_addr = "{}:{}".format(get_url_from_ip(masterIP),
                            self.configs.saved_data.get("master_webserver_port"))
        try:
            timeout = 30
            masterXClusterApi = "http://{}/api/v1/xcluster".format(master_addr)
            Output.log("Trying to parse xcluster info from {}".format(masterXClusterApi) +
                        " (Timeout={})".format(timeout))
            response = urlopen(Request(masterXClusterApi), timeout=timeout)

            jsonResponse = json.load(response)
            if not jsonResponse:
                Output.log("Empty response from {}.".format(masterXClusterApi))
                return []

            Output.log("Got the response from {}. ".format(masterXClusterApi) +
                       "response: {}".format(jsonResponse))
            return jsonResponse
        except HTTPError as http_err:
            Output.log('HTTP error occurred while fetching current' +
                    'masters from tserver: {}'.format(http_err))
            return []
        except URLError as url_err:
            Output.log('URL error occurred while fetching current' +
                    'masters from tserver: {}'.format(url_err))
            return []
        except Exception as err:
            Output.log('Other error occurred while fetching current' +
                    'masters from tserver: {}'.format(err))
            return []


    # Returns a list of invalid IPs from the list of IPs provided.
    def get_invalid_ips(self, ips):
        invalid_ips = []
        for ip in ips:
            if not self.validate_hostname_ip(ip):
                invalid_ips.append(ip)

        return invalid_ips

    def find_ip_address_of_node(self):

        host_name = self.find_hostname_of_node()
        host_ip = socket.gethostbyname(host_name)

        return host_ip

    def find_hostname_of_node(self):
        fqdn = socket.getfqdn()

        if "localhost" in fqdn:
            fqdn = socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET,
                                  socket.SOCK_DGRAM, socket.IPPROTO_IP,
                                  socket.AI_CANONNAME)[0][3]
            if "localhost" in fqdn:
                fqdn = socket.gethostname()

        return fqdn

    # Check whether the leader master is secure.
    def is_leader_master_secure(self, master_hostport, timeout=60):
        start_time = time.time()
        now = start_time
        try_count = 0
        while True:
            try:
                try_count+=1
                leaderMasterURL = "http://{}/api/v1/varz".format(master_hostport)
                Output.log("Trying to get response from {}. Try Count: {}".format(leaderMasterURL,
                                                                                    try_count))
                response = urlopen(Request(leaderMasterURL), timeout=timeout)
                jsonResponseFromMaster = json.load(response)
                listOfAllFlags = jsonResponseFromMaster.get("flags")

                encryptionFlag = [flag for flag in listOfAllFlags if
                    flag.get("name") == "use_node_to_node_encryption"]

                return encryptionFlag[0].get("value") == "true"

            except HTTPError as http_err:
                Output.log_error_and_exit("HTTP error occurred while checking for security of " +
                                        "leader master: {}".format(http_err))
            except Exception as err:
                now = time.time()
                if now - start_time > timeout:
                    Output.log_error_and_exit("Other error occurred while checking for " +
                        "security of leader master: {}. Timeout: {}".format(err, now - start_time))
                else:
                    Output.log("Other error occurred while checking for " +
                            "security of leader master: {}.".format(err))
                    time.sleep(0.2)


    def handle_config_files(self, args, conf_dir, config_file, base_dir):
        if not os.path.isdir(conf_dir):
            if args.parser == "start":
                os.makedirs(conf_dir)

        if os.path.exists(config_file):
            # Load already saved configs if yugabyted.conf exists
            self.configs = Configs.parse_config_file(config_file, base_dir)
        else:
            # Initialise configs
            self.configs = Configs(config_file, base_dir)
            if args.parser == "start":
                open(config_file, 'a').close()

        if args.parser == "start" and args.config:
            args.config = \
                os.path.realpath(
                os.path.abspath(
                os.path.expanduser(args.config)))
            user_configs = Configs.parse_user_config_file(args.config)
            # User should not be able to override data_dir
            if (self.configs.saved_data.get("cluster_member")
                and args.data_dir is None and user_configs.get("data_dir")):
                user_configs_data_dir = [
                    path for path in user_configs.get("data_dir").split(',')
                ]
                config_data_dir = [self.configs.saved_data.get("data_dir")] + \
                  [x for x in self.configs.saved_data.get(
                    "additional_data_dir").split(',') if x]
                # Existing data directories cannot be removed or changed
                # through changing the user config file during restart
                if len(user_configs_data_dir) < len(config_data_dir) or \
                    user_configs_data_dir[:len(config_data_dir)] != config_data_dir:
                    Output.print_out(
                        "Data directory already exists at: {}.".format(
                                                ", ".join(config_data_dir))
                    )
                    sys.exit(1)

            data_dir = self.configs.saved_data.get("data_dir")
            additional_data_dir = self.configs.saved_data.get(
                                            "additional_data_dir")
            # Update saved configs with user configurations
            self.configs.saved_data.update(user_configs)
            # If args.data_dir is given then do not
            # change the data_path with user configs
            if (args.data_dir is not None and
                        self.configs.saved_data.get("cluster_member")):
                self.configs.saved_data["data_dir"] = data_dir
                self.configs.saved_data["additional_data_dir"] = \
                                                    additional_data_dir

        return self.configs
    # Parse the config file and input parent flags to validate and set them.
    # Parent flags: --data_dir, --log_dir
    def validate_and_set_parent_configs(self, args):

        home_dir = os.path.expanduser("~")
        default_base_dir = os.path.join(home_dir, "var")
        default_conf_path = os.path.join(default_base_dir, "conf", "{}.conf".format(SCRIPT_NAME))
        base_dir = os.path.abspath(os.path.expanduser(args.base_dir)) if args.base_dir \
            else default_base_dir
        base_dir_conf = ''
        if args.base_dir:
            base_dir_conf = os.path.join(base_dir, "conf", "{}.conf".format(SCRIPT_NAME))

        has_errors = False

        self.conf_file = base_dir_conf or \
                         Configs.get_brew_config() or default_conf_path
        self.conf_file = os.path.realpath(os.path.expanduser(self.conf_file))

        conf_dir = os.path.dirname(self.conf_file)
        self.configs = self.handle_config_files(args, conf_dir, self.conf_file, base_dir)

        if self.configs.saved_data.get("cluster_member"):
            if args.data_dir is not None:
                data_dir_list = [
                    os.path.abspath(os.path.realpath(os.path.expanduser(path)))
                    for path in args.data_dir.split(',')
                ]
                config_data_dir = [self.configs.saved_data.get("data_dir")] + \
                  [x for x in self.configs.saved_data.get(
                    "additional_data_dir").split(',') if x]
                # Existing data directories cannot be removed or changed
                if len(data_dir_list) < len(config_data_dir) or \
                    data_dir_list[:len(config_data_dir)] != config_data_dir:
                    has_errors = True
                    Output.print_out(
                        "Data directory already exists at: {}.".format(
                                                ", ".join(config_data_dir))
                    )
            if args.log_dir is not None:
                args.log_dir = os.path.abspath(os.path.realpath(
                                                os.path.expanduser(args.log_dir)))
                config_log_dir = self.configs.saved_data.get("log_dir")
                # Existing log directory can be changed
                if config_log_dir != args.log_dir:
                    Output.print_out(
                        "Old log directory already exists at {}. New logs will go to {}".format(
                            config_log_dir, args.log_dir))

        if has_errors:
            sys.exit(1)

        parent_flags = ["log_dir", "data_dir"]
        # Override configs and defaults with user specified variables
        for k, v in get_kv(args.__dict__):
            if (v is not None and k in parent_flags and k in self.configs.saved_data):
                self.configs.saved_data[k] = v

        data_dir = self.configs.saved_data["data_dir"]
        data_dir_paths = data_dir.split(',')
        self.configs.saved_data["data_dir"] = data_dir_paths[0]
        if len(data_dir_paths) > 1:
            self.configs.saved_data["additional_data_dir"] = \
                                            ','.join(data_dir_paths[1:])
        log_dir = self.configs.saved_data["log_dir"]
        self.configs.save_configs()

        for dir_path in data_dir_paths:
            if not os.path.isdir(dir_path):
                os.makedirs(dir_path)
        if not os.path.isdir(log_dir):
            os.makedirs(log_dir)

        self.script = ScriptProcessManager(log_dir,
                            self.configs.saved_data.get("data_dir"))

    # Check and validate the flags regarding the secure deployment of the node.
    # Check whether the certs files are present or not.
    # Create certs if it is 1-node cluster throw error otherwise.
    def validate_security_configs(self, args):
        home_dir = os.path.expanduser("~")
        default_base_dir = os.path.join(home_dir, "var")
        base_dir = os.path.abspath(os.path.expanduser(args.base_dir)) if args.base_dir \
            else default_base_dir

        default_certs_dir = os.path.join(base_dir, "certs")

        certs_dir = self.configs.saved_data.get("certs_dir") or \
                    default_certs_dir

        has_errors = False

        certs_files = ["ca.crt", "node."+args.advertise_address+".crt",
                    "node."+args.advertise_address+".key"]

        if args.certs_dir is not None:
            args.certs_dir = os.path.expanduser(args.certs_dir)

            if not os.path.exists(args.certs_dir):
                # Case Scenario: When the directory provided by the user through
                # --certs_dir flag in CLI does not exists.
                has_errors = True
                Output.print_out(Output.make_red("ERROR") +
                    ": Trying to use {} as the certs directory. ".format(args.certs_dir) +
                    "No such directory found. Please provide the correct directory " +
                    "where the certificates are through --certs_dir flag.")
            else:
                if not check_files_in_path(args.certs_dir, certs_files):
                    # Case Scenario: When the directory provided by the user through --certs_dir
                    # flag in CLI exists but doesn't have the correct certs.
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") +
                        ": Trying to use {} as the certs ".format(args.certs_dir) +
                        "directory. Correct certs not found in this directory. Please " +
                        "either use 'cert generate_server_certs' command to create " +
                        "certificates or follow this doc to create your own " +
                        "certificates: {}".format(GENERATE_SERVER_CERTS))
                else:
                    # Case Scenario: When the directory provided by the user through
                    # --certs_dir flag in CLI exists and have the correct certs.
                    Output.log("Found certs at {}.".format(args.certs_dir))
        else:
            if os.path.exists(certs_dir):
                if not check_files_in_path(certs_dir, certs_files):
                    # Case Scenario: When the default certs directory or the certs directory saved
                    # in conf file exists but doesn't have the correct certs.
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") +
                        ": Trying to use {} as the certs ".format(certs_dir) +
                        "directory. Correct certs not found in this directory. Please " +
                        "either use 'cert generate_server_certs' command to create " +
                        "certificates or follow this doc to create your own " +
                        "certificates: {}".format(GENERATE_SERVER_CERTS))
                else:
                    # Case Scenario: When the default certs directory or the certs directory saved
                    # in conf file exists and has the correct certs.
                    Output.log("Found certs at {}.".format(certs_dir))
            else:
                if certs_dir != default_certs_dir:
                    # Case Scenario: When the certs directory is mentioned in the configs
                    # file but the directory doesn't exist.
                    # Example: When the user starts a secure cluster, then stops it.
                    # Then due to some reason the certs directory that was being used has
                    # been deleted. Now when the user tries to restart the node,
                    # yugabyted will try to use the same dir as it was stored in the config file,
                    # but the directory doesn't exist.
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") +
                        ": The directory mentioned in the --cert_dir value of the configs file " +
                        "doen't exist. Please create the directory and copy the certs into that " +
                        "directory.")
                elif (args.join or self.configs.saved_data.get("join")):
                    # Case Scenario: When the user is trying to start and join a node
                    # to a secure cluster but the generated certs are neither present
                    # in the default directory nor given through --certs_dir flag.
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") +
                        ": Trying to start a secure node but couldn't find the certs at " +
                        "default directory. Cannot create new certs as the --join flag " +
                        "was used. Please generate the node certificates in 1 node and " +
                        "copy them to other nodes.")
                else:
                    # Case Scenario: When the user is trying to start a secure node but
                    # the generated certs are neither present in default directory nor
                    # given through --certs_dir flag. Create new certs in this case.
                    Output.log("Couldn't find the certs so creating new certificates.")
                    if self.check_openssl():
                        has_errors = True
                        Output.print_out(Output.make_red("Error") + ": openssl " +
                            "not installed. Can't create certificates.")

                    if not has_errors:
                        gen_certs_dir = self.configs.saved_data.get("gen_certs_dir")
                        root_certs_dir = os.path.join(gen_certs_dir, "root-ca")

                        if not self.generate_ca_certs(root_certs_dir=root_certs_dir):
                            has_errors = True
                            Output.print_out(Output.make_red("Error") +
                                "Cert generation failed. Please check the logs.")

                        hostnames = [args.advertise_address]
                        generated_certs_hostnames = self.generate_node_server_certs(
                                            hostnames=hostnames, gen_certs_dir=gen_certs_dir)
                        if not has_errors and not len(generated_certs_hostnames):
                            has_errors = True
                            Output.print_out(Output.make_red("Error") +
                                "Cert generation failed. Please check the logs.")

                        if not has_errors:
                            os.makedirs(certs_dir)

                            for file in certs_files:
                                shutil.copy2(os.path.join(gen_certs_dir, args.advertise_address,
                                                            file), certs_dir)

            args.certs_dir = certs_dir



        if has_errors:
            sys.exit(1)

    # Find the correct security nature of the deployment of the node by checking
    # secure or insecure flags passed by the user and secure and insecure options
    # present in the config file
    def find_security_nature_of_deployment(self, args):
        if args.secure and not args.insecure:
            # If --secure flag is passed in the CLI and --insecure is not. Node should be
            # deployed in secure mode.
            args.secure = True
            args.insecure = False
            return

        if args.insecure and not args.secure:
            # If --insecure flag is passed in the CLI and --secure is not. Node should be
            # deployed in insecure mode.
            args.secure = False
            args.insecure = True
            return

        if args.secure and args.insecure:
            # If both --secure and --insecure flags are passed in the CLI, show an error.
            Output.log_error_and_exit(Output.make_red("ERROR") +
                ": --secure flag can't be used together with --insecure flag")

        if not self.configs.saved_data.get("secure"):
            # If neither --secure nor --insecure flag is passed in the CLI and the value
            # of --secure in conf file is False then start the node in insecure mode
            args.secure = False
            args.insecure = True
            return

        if not self.configs.saved_data.get("insecure"):
            # If neither --secure nor --insecure flag is passed in the CLI and value of
            # --insecure in conf file is False and --secure is True then start
            # the node in secure mode
            args.secure = True
            args.insecure = False
            return

        # If neither --secure nor --insecure flag and both --secure and --insecure have
        # values in conf file have value as True, show an error.
        Output.log_error_and_exit(Output.make_red("ERROR") +
            ": config file {} has both secure and ".format(self.conf_file) +
            "insecure set to True. Please change one of them to False")

    # Get the current master leader known to a tserver using api/v1/masters endpoint
    # of the tserver.
    def get_current_master_leader_from_api(self, tserverIP, tserver_webserver_port, timeout=60):
        tserver_addr = "{}:{}".format(get_url_from_ip(tserverIP),
                                      tserver_webserver_port)
        try:
            tserverMastersAPI = "http://{}/api/v1/masters".format(tserver_addr)
            Output.log("Trying to get masters information from {}".format(tserverMastersAPI) +
                        " (Timeout={})".format(timeout))
            response = urlopen(Request(tserverMastersAPI), timeout=timeout)
            jsonResponseFromTserver = json.load(response)
            dictOfAllNodes = jsonResponseFromTserver.get("master_server_and_type")

            masterLeader = ""
            for node in dictOfAllNodes:
                if node["is_leader"]:
                    masterLeader = urlparse("//" + node["master_server"]).hostname
                    break

            Output.log("Tserver {} returned the following".format(tserverIP) +
                        "master leader {}.".format(masterLeader),
                            logging.DEBUG)

            return masterLeader

        except HTTPError as http_err:
            Output.log("HTTP Error occured while hitting the api endpoint " +
                "http://{}/api/v1/masters: {}".format(tserver_addr, http_err))
            Output.log_error_and_exit(Output.make_red("ERROR:") + " Node at the join ip " +
                "provided is not reachable.")
        except URLError as url_error:
            Output.log("Some error occured while hitting the api endpoint " +
                "http://{}/api/v1/masters: {}".format(tserver_addr, url_error))
            Output.log_error_and_exit(Output.make_red("ERROR:") + " Node at the join ip " +
                "provided is not reachable.")
        except Exception as err:
            Output.log("Error occured while hitting the api endpoint " +
                "http://{}/api/v1/masters: {}".format(tserver_addr, err))
            Output.log_error_and_exit(Output.make_red("ERROR:") + " Node at the join ip " +
                "provided is not reachable.")



    def validate_ybc_cloud_uri(self, cloud_storage_uri):

        if cloud_storage_uri.startswith("s3://"):
            pattern = r'^s3://[a-zA-Z0-9._-]+(/[a-zA-Z0-9._-]+)*$'
            if re.match(pattern, cloud_storage_uri) is not None:

                if os.environ.get("AWS_ACCESS_KEY_ID") is None or \
                   os.environ.get("AWS_SECRET_ACCESS_KEY") is None or \
                   os.environ.get("AWS_DEFAULT_REGION") is None:

                    Output.print_and_log(
                        Output.make_red("Error: ") + "AWS credentials are not provided. " + \
                            "Environment variables AWS_ACCESS_KEY_ID, " + \
                            "AWS_SECRET_ACCESS_KEY and AWS_DEFAULT_REGION " + \
                            "needs to be set."
                    )
                    return False

                self.configs.temp_data["ybc_cloud_type"] = "s3"
                bucket_dir = cloud_storage_uri[5:].split("/", 1)
                # s3 bucket
                self.configs.temp_data["ybc_cloud_storage_bucket"] = bucket_dir[0]
                # s3 directory
                self.configs.temp_data["ybc_cloud_storage_dir"] = bucket_dir[1]
                return True
        elif cloud_storage_uri.startswith("gs://"):
            pattern = r'^gs://[a-zA-Z0-9._-]+(/[a-zA-Z0-9._-]+)*$'
            if re.match(pattern, cloud_storage_uri) is not None:
                self.configs.temp_data["ybc_cloud_type"] = "google"
                bucket_dir = cloud_storage_uri[5:].split("/", 1)
                # gcp bucket
                self.configs.temp_data["ybc_cloud_storage_bucket"] = bucket_dir[0]
                # gcp directory
                self.configs.temp_data["ybc_cloud_storage_dir"] = bucket_dir[1]
                return True
        elif cloud_storage_uri.find('blob.core.windows.net') !=  -1:
            Output.print_and_log(
                Output.make_red("Error: ") + "Azure support is comming soon!"
            )
            return False
        else:
            # check if nfs path is valid
            pattern = r'^/([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+$'
            if re.match(pattern, cloud_storage_uri) is not None:
                if not os.environ.get("YBC_NFS_DIR"):
                    os.environ["YBC_NFS_DIR"] = YBC_NFS_DIR
                    if not os.path.exists(YBC_NFS_DIR):
                        os.mkdir(YBC_NFS_DIR)
                self.configs.temp_data["ybc_cloud_type"] = "nfs"
                bucket_dir = cloud_storage_uri[0:].split("/")[-2:]
                self.configs.temp_data["ybc_cloud_storage_bucket"] = bucket_dir[0]
                self.configs.temp_data["ybc_cloud_storage_dir"] = bucket_dir[1]
                return True
            return False

    def validate_ybc_args(self, args):

        if args.cloud_storage_uri is not None:
            if not self.validate_ybc_cloud_uri(args.cloud_storage_uri):
                Output.print_and_log(
                    Output.make_red("Error: ") + "--cloud_storage_uri is invalid."
                )
                return False
        else:
            Output.print_and_log(
                Output.make_red("Error: ") + "--cloud_storage_uri is empty."
            )
            return False

        if args.status:
            self.configs.temp_data["ybc_status"] = True

        if args.database is not None and args.keyspace is not None:
            Output.print_and_log(
                    Output.make_red("Error: ") +
                    "Cannot specify both database and keyspace args. " +
                    "Specify either the database or the keyspace argument for the command."
                )
            return False
        elif args.database is None and args.keyspace is None:
            Output.print_and_log(
                Output.make_red("Error: ") +
                "Specify either the database or the keyspace argument for the command."
            )
            return False
        elif args.database is not None and args.keyspace is None:
            self.configs.temp_data["database_name_backup_restore"] = args.database
            return True
        elif args.keyspace is not None and args.database is None:
            self.configs.temp_data["keyspace_name_backup_restore"] = args.keyspace
            return True

        return False

    # Validate time format
    def validate_time_format(self, restore_time):
        try:
            # Check for value errors
            datetime.strptime(restore_time, "%Y-%m-%d %I:%M:%S %p")
            # Split the restore_time string to directly check the length of each component.
            date, time = restore_time.split(' ')[0:2]
            _, month, day = date.split('-')
            hour, minute, second = time.split(':')

            for component in [month, day, hour, minute, second]:
                if len(component) != 2:
                    return False

            return True

        except ValueError:
        # The time string does not match the expected format.
            return False

    def flags_not_provided(self, args, required_flags):
        flags_not_provided_list = list()
        provided_flags = [provided_flag.split('=')[0] for  provided_flag in sys.argv]
        for flag in required_flags:
            if "--" + flag not in provided_flags:
                flags_not_provided_list.append(flag)

        return flags_not_provided_list

    # Parse config file and input args. Validate them and save any new configs.
    def validate_and_set_configs(self, args):

        has_errors = False

        if self.configs.saved_data.get("database_password") is not None:
            self.setup_env_init.update_passwords(self.configs.saved_data.get("database_password"))

        if args.parser == "create_checkpoint":
            required_flags = ["replication_id", "databases"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                    "Please provide them while running `yugabyted xcluster create_checkpoint`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
            databases = args.databases.split(",")
            db_not_exist_list = list()
            for db in databases:
                if not ysql_proxy.db_exists(db):
                    db_not_exist_list.append(db)

            if len(db_not_exist_list):
                if len(db_not_exist_list) == 1:
                    err_msg = "database {}".format(db_not_exist_list[0])
                else:
                    err_msg = "databases {}".format(','.join(db_not_exist_list))
                Output.log_error_and_exit(Output.make_red("Error") + ": " + err_msg +
                    " does not exists in this cluster. Please create " +
                    "the schema for the database to be replicated before running " +
                    "`yugabyted xcluster create_checkpoint`.")

            self.configs.temp_data["xcluster_replication_id"] = args.replication_id
            self.configs.temp_data["xcluster_databases"] = args.databases
            self.configs.temp_data["xcluster_automatic_mode"] = args.automatic_mode

        if args.parser == "add_to_checkpoint":
            required_flags = ["replication_id", "databases"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                    "Please provide them while running `yugabyted xcluster add_to_checkpoint`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            if not self.does_replication_id_exists(args.replication_id):
                Output.log_error_and_exit(Output.make_red("Error") + ": --replication_id" +
                        " {} provided is not present. ".format(args.replication_id) +
                        "Was the xcluster replication with this replication_id set-up?")

            ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
            databases = args.databases.split(",")
            db_not_exist_list = list()
            db_exist_in_replication = list()
            for db in databases:
                if not ysql_proxy.db_exists(db):
                    db_not_exist_list.append(db)
                if self.does_database_exist_in_replication(db, args.replication_id):
                    db_exist_in_replication.append(db)

            if len(db_not_exist_list):
                if len(db_not_exist_list) == 1:
                    err_msg = "database {}".format(db_not_exist_list[0])
                else:
                    err_msg = "databases {}".format(','.join(db_not_exist_list))
                Output.log_error_and_exit(Output.make_red("Error") + ": " + err_msg +
                    " does not exists in this cluster. Please create " +
                    "the schema for the database to be replicated before running " +
                    "`yugabyted xcluster create_checkpoint`.")

            if len(db_exist_in_replication):
                if len(db_exist_in_replication) == 1:
                    err_msg = "database {}".format(db_exist_in_replication[0])
                else:
                    err_msg = "databases {}".format(','.join(db_exist_in_replication))
                Output.log_error_and_exit(Output.make_red("Error") + ": " + err_msg +
                    " already exists in the replication {}. ".format(args.replication_id) +
                    "Cannot add these again.")

            self.configs.temp_data["xcluster_replication_id"] = args.replication_id
            self.configs.temp_data["xcluster_databases"] = args.databases

        if args.parser == "set_up":
            required_flags = ["replication_id", "target_address", "bootstrap_done"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                    "Please provide them while running `yugabyted xcluster set_up`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            if not self.does_replication_id_exists(args.replication_id):
                Output.log_error_and_exit(Output.make_red("Error") + ": --replication_id" +
                        " {} provided is not present. ".format(args.replication_id) +
                        "Was the `yugabyted xcluster create_checkpoint` command run?")

            if self.validate_hostname_ip(args.target_address):
                target_master_addresses = self.get_current_masters_from_api(args.target_address)
                if target_master_addresses == '':
                    Output.log_error_and_exit(Output.make_red("Error") + ": cannot reach node at " +
                    "provided target IP {}. Is target cluster running?".format(args.target_address))
                else:
                    self.configs.temp_data["xcluster_target_addresses"] = target_master_addresses
            else:
                Output.log_error_and_exit(Output.make_red("Error") + ": --target_address" +
                        " {} provided is not a valid address. ".format(args.target_address) +
                        "Please try again with a valid IPV4, IPV6 or DNS.")

            self.configs.temp_data["xcluster_replication_id"] = args.replication_id
            self.configs.temp_data["xcluster_bootstrap_done"] = args.bootstrap_done

        if args.parser == "add_to_replication":
            required_flags = ["replication_id", "target_address", "bootstrap_done", "databases"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                    "Please provide them while running `yugabyted xcluster add_to_replication`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            if not self.does_replication_id_exists(args.replication_id):
                Output.log_error_and_exit(Output.make_red("Error") + ": --replication_id" +
                        " {} provided is not present. ".format(args.replication_id) +
                        "Was the xcluster replication with this replication_id set-up?")

            ysql_proxy = YsqlProxy(self.advertise_ip(), self.configs.saved_data.get("ysql_port"))
            databases = args.databases.split(",")
            db_not_exist_list = list()
            db_not_exist_in_replication = list()
            for db in databases:
                if not ysql_proxy.db_exists(db):
                    db_not_exist_list.append(db)
                if not self.does_database_exist_in_replication(db, args.replication_id):
                    db_not_exist_in_replication.append(db)

            if len(db_not_exist_list):
                if len(db_not_exist_list) == 1:
                    err_msg = "database {}".format(db_not_exist_list[0])
                else:
                    err_msg = "databases {}".format(','.join(db_not_exist_list))
                Output.log_error_and_exit(Output.make_red("Error") + ": " + err_msg +
                    " does not exists in this cluster. Please create " +
                    "the schema for the database(s) to be added to the replication and run " +
                    "`yugabyted xcluster add_to_checkpoint` to checkpoint the database(s) before " +
                    "running `yugabyted xcluster add_to_replication`")

            if len(db_not_exist_in_replication):
                if len(db_not_exist_in_replication) == 1:
                    err_msg = "database {}".format(db_not_exist_in_replication[0])
                else:
                    err_msg = "databases {}".format(','.join(db_not_exist_in_replication))
                Output.log_error_and_exit(Output.make_red("Error") + ": " + err_msg +
                    " does not exist in the replication {}. ".format(args.replication_id) +
                    "Was `yuagbyted xcluster add_to_checkpoint` run for this database(s)?")

            if self.validate_hostname_ip(args.target_address):
                target_master_addresses = self.get_current_masters_from_api(args.target_address)
                if target_master_addresses == '':
                    Output.log_error_and_exit(Output.make_red("Error") + ": cannot reach node at " +
                    "provided target IP {}. Is target cluster running?".format(args.target_address))
                else:
                    self.configs.temp_data["xcluster_target_addresses"] = target_master_addresses
            else:
                Output.log_error_and_exit(Output.make_red("Error") + ": --target_address" +
                        " {} provided is not a valid address. ".format(args.target_address) +
                        "Please try again with a valid IPV4, IPV6 or DNS.")

            self.configs.temp_data["xcluster_replication_id"] = args.replication_id
            self.configs.temp_data["xcluster_bootstrap_done"] = args.bootstrap_done
            self.configs.temp_data["xcluster_databases"] = args.databases

        if args.parser == "delete_replication":
            required_flags = ["replication_id", "target_address"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                    "Please provide them while running `yugabyted xcluster delete_replication`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            if self.validate_hostname_ip(args.target_address):
                self.configs.temp_data["xcluster_target_addresses"] = args.target_address
            else:
                Output.log_error_and_exit(Output.make_red("Error") + ": --target_address" +
                        " {} provided is not a valid address. ".format(args.target_address) +
                        "Please try again with a valid IPV4, IPV6 or DNS.")

            if not self.does_replication_id_exists(args.replication_id):
                Output.log_error_and_exit(Output.make_red("Error") + ": --replication_id" +
                        " {} provided is not present. ".format(args.replication_id) +
                        "Was the xcluster replication with this replication_id set-up?")

            self.configs.temp_data["xcluster_replication_id"] = args.replication_id

        if args.parser == "remove_database_from_replication":
            required_flags = ["replication_id", "target_address", "databases"]
            flags_not_provided_list = self.flags_not_provided(args, required_flags)

            if len(flags_not_provided_list) != 0:
                err_msg = Output.make_red("Error") + ": The following flags are not provided. " +\
                                            "Please provide them while running " +\
                                            "`yugabyted xcluster remove_database_from_replication`."
                for flag in flags_not_provided_list:
                    err_msg += "\n- " + flag
                Output.print_out(err_msg)
                sys.exit(1)

            if self.validate_hostname_ip(args.target_address):
                self.configs.temp_data["xcluster_target_addresses"] = args.target_address
            else:
                Output.log_error_and_exit(Output.make_red("Error") + ": --target_address" +
                        " {} provided is not a valid address. ".format(args.target_address) +
                        "Please try again with a valid IPV4, IPV6 or DNS.")

            if not self.does_replication_id_exists(args.replication_id):
                Output.log_error_and_exit(Output.make_red("Error") + ": --replication_id" +
                        " {} provided is not present. ".format(args.replication_id) +
                        "Was the xcluster replication with this replication_id set-up?")

            self.configs.temp_data["xcluster_databases"] = args.databases
            self.configs.temp_data["xcluster_replication_id"] = args.replication_id

        if args.parser == "status" and sys.argv[1] == "xcluster":
            if args.replication_id is not None:
                self.configs.temp_data["xcluster_replication_id"] = args.replication_id

        if args.parser == "backup":
            if not self.validate_ybc_args(args):
                has_errors = True

        if args.parser == "restore":
            if not args.cloud_storage_uri and not args.recover_to_point_in_time:
                Output.print_out(Output.make_red("Error") + ": Specify either " +
                    "--cloud_storage_uri or --recover_to_point_in_time " +
                    "to perform a restore operation.")
                sys.exit(1)

            # PITR operation
            if args.recover_to_point_in_time is not None:

                restore_error = []

                if not self.validate_time_format(args.recover_to_point_in_time):
                    restore_error.append(
                    "* Invalid time format specified for --recover_to_point_in_time flag: {}. " \
                    "Please follow the format: 'YYYY-MM-DD HH:MM:SS AM/PM'." \
                                    .format(args.recover_to_point_in_time)
                    )
                if not (args.database or args.keyspace):
                    restore_error.append(
                    "* Specify either database or keyspace to" \
                    " restore to a point in time."
                    )
                if args.database and args.keyspace:
                    restore_error.append(
                    "* Cannot specify both database and keyspace args." \
                    " Please specify either database or keyspace name."
                    )
                if (args.database in SYSTEM_NAMESPACES) or (args.keyspace in SYSTEM_NAMESPACES):
                        restore_error.append(
                        "* Cannot specify system namespaces to recover to a point" \
                        " in time. Please specify a non-system namespace."
                    )
                if args.cloud_storage_uri:
                    restore_error.append(
                        "* --recover_to_point_in_time flag can't be used with " \
                        "--cloud_storage_uri flag. Please remove the --cloud_storage_uri flag" \
                        " when specifying the --recover_to_point_in_time flag."
                    )
                if args.status:
                    restore_error.append(
                        "* --recover_to_point_in_time flag can't be used with " \
                        "--status flag. Please remove the --status flag" \
                        " when specifying the --recover_to_point_in_time flag."
                    )
                if restore_error:
                    has_errors = True
                    Output.print_out(Output.make_red("Error") + ": Following errors found" +
                                " while restoring to a point in time:\n" +
                                "\n".join(restore_error))
                # No errors found in PITR restore args
                else:
                    self.configs.temp_data["database_name_backup_restore"] = args.database
                    self.configs.temp_data["keyspace_name_backup_restore"] = args.keyspace
                    self.configs.temp_data["restore_time"] = args.recover_to_point_in_time
            else:
                if not self.validate_ybc_args(args):
                    has_errors = True

        if args.parser == "collect_logs":
            self.configs.temp_data["collect_logs_stdout"] = args.stdout
            if args.collect_at_dir is not None:
                args.collect_at_dir = \
                os.path.realpath(
                os.path.abspath(
                os.path.expanduser(args.collect_at_dir)))
                self.configs.temp_data["collect_at_dir"]=args.collect_at_dir

            collect_at_dir=self.configs.temp_data.get("collect_at_dir")
            if not os.path.isdir(collect_at_dir):
                os.makedirs(collect_at_dir)

        if args.parser == "point_in_time_recovery":
            # All three action flags namely --enable,--disable
            # and --status must be mutually exclusive
            if not args.enable and not args.disable and not args.status:
                has_errors = True
                Output.print_and_log(
                    Output.make_red("Error") +
                    ": Specify either the --enable, --disable or --status flag."
                )
            elif args.enable and args.disable and args.status:
                has_errors = True
                Output.print_and_log(
                    Output.make_red("Error") +
                    ": --enable, --disable and --status flags can't be used together."
                )
            elif args.enable and args.disable:
                has_errors = True
                Output.print_and_log(
                    Output.make_red("Error") +
                    ": --enable flag can't be used together with --disable flag."
                )
            elif args.enable and args.status:
                has_errors = True
                Output.print_and_log(
                    Output.make_red("Error") +
                    ": --enable flag can't be used together with --status flag."
                )
            elif args.disable and args.status:
                has_errors = True
                Output.print_and_log(
                    Output.make_red("Error") +
                    ": --disable flag can't be used together with --status flag."
                )
            # Validation checks for each PITR action
            else:
                pitr_error_map = {
                        'enable': [],
                        'disable': [],
                        'status': [],
                    }
                if args.enable:
                    action = 'enable'
                    self.configs.temp_data["enable_pitr"] = args.enable
                elif args.disable:
                    action = 'disable'
                    self.configs.temp_data["disable_pitr"] = args.disable
                elif args.status:
                    action = 'status'

                # Case Scenario: When database or keyspace is not specified
                # with --enable or --disable flag
                if not (args.database or args.keyspace) and action != 'status':
                    pitr_error_map[action].append(
                    "* Specify either database or keyspace to " \
                    "{} point-in-time recovery.".format(action)
                )
                # Case Scenario: When both database and keyspace are specified
                if args.database and args.keyspace:
                        pitr_error_map[action].append(
                        "* Cannot specify both database and keyspace args." \
                        " Please specify either database or keyspace name."
                        )
                # Case Scenario: When a system namespace is specified
                if (args.database in SYSTEM_NAMESPACES) or (args.keyspace in SYSTEM_NAMESPACES):
                        pitr_error_map[action].append(
                        "* Cannot specify system namespaces to configure point" \
                        "-in-time recovery. Please specify a non-system namespace."
                        )

                if args.retention:
                    # Case Scenario: When retention flag is specified
                    # with --disable or status flag
                    if action in ["disable", "status"]:
                        pitr_error_map[action].append(
                        "* --retention flag can't be used with --{} flag." \
                        " Please remove the --retention flag when specifying" \
                        " the --{} flag.".format(action, action)
                        )
                    elif not args.retention.isdigit() or int(args.retention) < 2:
                        pitr_error_map[action].append(
                        "* Retention period must be a positive integer greater than 1." \
                        " Please specify a valid postive integer greater than 1 to" \
                        " enable point-in-time recovery."
                        )

                # List errors found while configuring pitr for a YugabyteDB cluster
                if pitr_error_map.get("enable"):
                    has_errors = True
                    Output.print_out(Output.make_red("Error") + ": Following errors found" +
                                " while enabling point-in-time recovery:\n" +
                                "\n".join(pitr_error_map["enable"]))
                if pitr_error_map.get("disable"):
                    has_errors = True
                    Output.print_out(Output.make_red("Error") + ": Following errors found" +
                                " while disabling point-in-time recovery:\n" +
                                "\n".join(pitr_error_map["disable"]))
                if pitr_error_map.get("status"):
                    has_errors = True
                    Output.print_out(Output.make_red("Error") + ": Following errors found" +
                                " while displaying the point-in-time recovery status:\n" +
                                "\n".join(pitr_error_map["status"]))

                # If no errors are found save the configs
                if not has_errors:
                    if args.database:
                        self.configs.temp_data["database_name_backup_restore"] = args.database
                    if args.keyspace:
                        self.configs.temp_data["keyspace_name_backup_restore"] = args.keyspace
                    if args.retention:
                        self.configs.temp_data["retention_period"] = int(args.retention)

        if args.parser == "admin_operation":
            if not args.command:
                has_errors = True
                Output.print_and_log(Output.make_red("Error:") + " --command flag is empty. " +
                                 "Please specify a yb-admin command to execute.")
            else:
                self.configs.temp_data["admin_command"] = args.command

            if args.master_addresses is not None:
                self.configs.temp_data[
                    "admin_operation_master_addresses"] = args.master_addresses

        if args.parser == "finalize_upgrade":
            if args.upgrade_ysql_timeout is not None:
                # Check if timeout is a digit and its integer value is greater than 0
                if not args.upgrade_ysql_timeout.isdigit() or \
                                            int(args.upgrade_ysql_timeout) <= 0:
                    has_errors = True
                    Output.print_and_log(Output.make_red("Error") + ": " +
                    "--upgrade_ysql_timeout value must be" +
                    " a positive integer greater than 0 in milliseconds." +
                    " Please specify a valid positive integer to set the" +
                    " YSQL upgrade timeout value.")
                else:
                    # Valid timeout value
                    self.configs.temp_data["upgrade_ysql_timeout"] = \
                                                    int(args.upgrade_ysql_timeout)

        if args.parser == "data_placement":
            if args.fault_tolerance is not None:
                if args.fault_tolerance.lower() in FAULT_TOLERANCE_CHOICES:
                    self.configs.temp_data["fault_tolerance"] = args.fault_tolerance.lower()
                else:
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") + ": " +
                        "Incorrect fault_tolerance value specified. " +
                        "Please specify one of the following - zone, region or cloud ")

            if args.constraint_value is not None:
                placement_info,priority_info = self.parse_constraint_value(args.constraint_value)
                placement_info = placement_info.split(",")

                # Initializing a map to store different types of errors
                constraint_value_error_map = {
                    'placement_error': [],
                    'invalid_priority_error': [],
                    'inconsistent_priority_error': [],
                    'missing_priority_error': False,
                }
                has_errors = False
                for constraint in placement_info:
                    is_valid_placement_info = True
                    cloud_info = constraint.split(".")
                    # Each cloud location should have exactly 3 parts:
                    # cloudprovider, region and zone
                    if (len(cloud_info) == 3):
                        for value in cloud_info:
                            if not value:
                                is_valid_placement_info = False
                    else:
                        is_valid_placement_info = False

                    if not is_valid_placement_info:
                        error_info = " * {}".format(constraint)
                        constraint_value_error_map['placement_error'].append(error_info)

                # Validate the format and consistency of the priority info
                if priority_info:
                    priority_numbers = []
                    zones = {}
                    for preference in priority_info:
                        zone, priority = preference.rsplit(":", 1)
                        # Ensure the priority value is a valid integer
                        if not priority.isdigit():
                            if priority == '':
                                error_info = " * Empty priority value found for {}".format(zone)
                            else:
                                error_info = " * {} in {}".format(priority, zone)
                            constraint_value_error_map['invalid_priority_error'].append(error_info)
                            continue

                        priority = int(priority)
                        priority_numbers.append(priority)
                        # Check that same zone must be specified with consistent priority values
                        if zone in list(zones.keys()):
                            if priority not in zones[zone]:
                                zones[zone].append(priority)
                        else:
                            zones[zone] = [priority]

                    for zone, priorities in zones.items():
                        # Condition to check if more than 2 priorities
                        # are associated with a specified zone
                        if len(priorities) > 1:
                            error_info = " * {} in {}". \
                            format(', '.join(map(str, sorted(priorities))), zone)
                            constraint_value_error_map['inconsistent_priority_error']. \
                            append(error_info)

                    # Ensure there are no missing priorities in the sequence
                    priority_set = set(priority_numbers)
                    max_priority = max(priority_numbers)
                    for i in range(1, max_priority + 1):
                        if i not in priority_set:
                            constraint_value_error_map['missing_priority_error'] = True
                            break

                errors = []

                # Add placement errors to the errors list
                if constraint_value_error_map.get('placement_error'):
                    errors.append("- Incorrect value specified for cloud location. " \
                                  "Please follow the format - cloudprovider.region.zone" \
                                  " for the following cloud locations:\n" + \
                        "\n".join(constraint_value_error_map['placement_error'])
                    )
                    has_errors = True
                # Add invalid priority errors to the errors list
                if constraint_value_error_map.get('invalid_priority_error'):
                    errors.append("- Incorrect priority value specified. " \
                        "Please specify a valid integer for the following cloud locations:\n" +
                        "\n".join(constraint_value_error_map['invalid_priority_error'])
                    )
                    has_errors = True
                # Add inconsistent priority errors to the errors list
                if constraint_value_error_map.get('inconsistent_priority_error'):
                    errors.append("- Same zone specified with different priority values. Please " \
                    "specify a consistent priority value for the following cloud locations:\n" +
                    "\n".join(constraint_value_error_map['inconsistent_priority_error'])
                    )
                    has_errors = True
                # Add missing priority errors to the errors list
                if constraint_value_error_map.get('missing_priority_error'):
                    errors.append("- Priority values should not skip numbers." \
                            " Please specify consecutive priority numbers."
                    )
                    has_errors = True

                if has_errors:
                    Output.print_out(Output.make_red("Error") + ": Following errors found for" \
                                        " --constraint_value flag:\n" + "\n".join(errors))
                 # If no errors are found, set the valid constraint value
                else:
                    self.configs.temp_data[
                        "constraint_value"] = args.constraint_value

            if args.rf is not None:
                self.configs.temp_data["replication_factor"] = str(args.rf)
            else:
                self.configs.temp_data["replication_factor"] = str("3")

        if args.parser == "generate_server_certs":
            if args.hostnames is None:
                has_errors = True
                Output.print_and_log(Output.make_red("Error") + ": Please provide the " +
                    "--hostnames along with \'yugabyted cert generate_server_certs\' command.")
            else:
                hostnames = args.hostnames.split(",")
                invalid_hostnames = self.get_invalid_ips(hostnames)
                if invalid_hostnames:
                    if len(invalid_hostnames) > 1:
                        msg = Output.make_red("ERROR") + ": Hostnames {} are not valid "\
                            "addresses. Please try again with a valid IPV4, IPV6 or DNS. "\
                            "hostnames.".format(",".join(invalid_hostnames))
                    else:
                        msg = Output.make_red("ERROR") + ": Hostname {} are not valid "\
                            "address. Please try again with a valid IPV4, IPV6 or DNS. "\
                            "hostnames.".format(",".join(invalid_hostnames))

                    has_errors = True
                    Output.print_and_log(msg)

            self.configs.temp_data["hostnames"] = args.hostnames

        if args.parser == "encrypt_at_rest":
            if not args.enable and not args.disable:
                has_errors = True
                Output.print_out(Output.make_red("ERROR") +
                    ": Either --enable or --disable flag has to be provided.")

            if args.enable and args.disable:
                has_errors = True
                Output.print_out(Output.make_red("ERROR") +
                    ": --enable and --disable flags cannot be used together.")
            elif args.enable:
                self.configs.temp_data["enable_encrypt_at_rest"] = args.enable
            elif args.disable:
                self.configs.temp_data["disable_encrypt_at_rest"] = args.disable

        if args.parser == "new":
            if args.data_placement_constraint is not None:
                data_constraints = args.data_placement_constraint.split(",")
                has_errors = False
                for constraint in data_constraints:
                    if ':' not in constraint:
                        has_errors = True
                    cloud_info = constraint.split(":")[0].split(".")
                    if (len(cloud_info) == 3):
                        for value in cloud_info:
                            if not value:
                                has_errors=True
                    else:
                        has_errors = True

                if has_errors:
                    Output.print_out(Output.make_red("ERROR") +
                        ": Incorrect value specified for --data_placement_constraint. " +
                        "Please specify comma sperated values with the num of replicas with " +
                        "format - <cloudprovider.region.zone>:<num_of_replicas>")
                else:
                    self.configs.temp_data[
                        "rr_data_placement_constraint"] = args.data_placement_constraint

            if args.rf is not None:
                if int(args.rf) > 0:
                    self.configs.temp_data["rr_replication_factor"] = str(args.rf)
                else:
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") + ": Please provide a valid " +
                                 "--rf flag. Replication factor cannot be less than 1.")

        if args.parser == "modify":
            if args.data_placement_constraint is None and args.rf is None:
                has_errors = True
                Output.print_out(Output.make_red("ERROR") + ": Please provide " +
                                 "either --data_placement_constraint or --rf flag " +
                                 "for modifying the read replica cluster.")

            if args.data_placement_constraint is not None:
                data_constraints = args.data_placement_constraint.split(",")
                has_errors = False
                for constraint in data_constraints:
                    if ':' not in constraint:
                        has_errors = True
                    cloud_info = constraint.split(".")
                    if (len(cloud_info) == 3):
                        for value in cloud_info:
                            if not value:
                                has_errors=True
                    else:
                        has_errors = True

                if has_errors:
                    Output.print_out(
                        "Incorrect value specified for --data_placement_constraint. " +
                        "Please specify comma sperated values with the num of replicas with " +
                        "format - <cloudprovider.region.zone>:<num_of_replicas>")
                else:
                    self.configs.temp_data[
                        "rr_data_placement_constraint"] = args.data_placement_constraint

            if args.rf is not None:
                if int(args.rf) > 0:
                    self.configs.temp_data["rr_replication_factor"] = str(args.rf)
                else:
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") + ": Please provide a valid " +
                                 "--rf flag. Replication factor cannot be less than 1.")

        if args.parser == "ysql":
            if args.username is not None:
                self.setup_env_init.set_ysql_user(args.username)
            if args.password is not None:
                self.setup_env_init.set_ysql_password(args.password)
            if args.database is not None:
                self.setup_env_init.set_ysql_db(args.database)

        if args.parser == "ycql":
            if args.username is not None:
                self.setup_env_init.set_ycql_user(args.username)
            if args.password is not None:
                self.setup_env_init.set_ycql_password(args.password)
            if args.keyspace is not None:
                self.setup_env_init.set_ycql_keyspace(args.keyspace)

        if args.parser == "check_version_compatibility":
            if args.timeout is not None:
                self.configs.temp_data["upgrade_timeout"] = args.timeout

        if args.parser == "ysql_catalog":
            if args.timeout is not None:
                self.configs.temp_data["upgrade_timeout"] = args.timeout

        if args.parser == "finalize_new_version":
            if args.timeout is not None:
                self.configs.temp_data["upgrade_timeout"] = args.timeout

        if args.parser == "stop":

            if args.upgrade is not None:
                self.configs.saved_data["upgrade_in_progress"] = args.upgrade

        if args.parser == "start":
            master_webserver_port = self.configs.saved_data.get("master_webserver_port")
            if args.master_webserver_port is not None:
                master_webserver_port = args.master_webserver_port

            tserver_webserver_port = self.configs.saved_data.get("tserver_webserver_port")
            if args.tserver_webserver_port is not None:
                tserver_webserver_port = args.tserver_webserver_port
            if args.read_replica and not args.join:
                Output.print_out(Output.make_red("ERROR") + ": read_replica neds to be " +
                                 "started with --join flag.")

            if args.cloud_location is not None:
                cloud_location = args.cloud_location.split(".")
                if len(cloud_location) == 3:
                    self.configs.saved_data["cloud_provider"] = cloud_location[0]
                    self.configs.saved_data["cloud_region"] = cloud_location[1]
                    self.configs.saved_data["cloud_zone"] = cloud_location[2]
                else:
                    Output.print_out(Output.make_red("ERROR") +
                        ": Incorrect format used for flag --cloud_location. " +
                        "Please use cloud.region.zone format.")
                    has_errors = True

            if args.fault_tolerance is not None:
                if args.fault_tolerance == "cloud":
                    Output.log_error_and_exit("Cloud based fault tolerance is not supported yet.")
                if args.fault_tolerance.lower() in START_FAULT_TOLERANCE_CHOICES:
                    self.configs.saved_data["fault_tolerance"] = args.fault_tolerance.lower()
                else:
                    has_errors = True
                    Output.print_out(Output.make_red("ERROR") + ": " +
                        "Incorrect fault_tolerance value specified. " +
                        "Please specify one of the following - zone, region or cloud ")

            if args.listen is not None:
                if args.advertise_address is not None and args.listen != args.advertise_address:
                    Output.print_out(Output.make_red("ERROR") +
                        ": --listen and --advertise_address flags " +
                        "are same. --listen is depricated. Can't have different values for them.")
                    has_errors = True
                args.advertise_address = args.listen

            if args.advertise_address is None:
                if not self.configs.saved_data.get("advertise_address"):
                    if os.path.exists("/.dockerenv"):
                            args.advertise_address = self.find_hostname_of_node()
                    else:
                        if OS_NAME == "Linux":
                            # Case Scenario: When advertise_address has no value in conf file
                            # and the OS is Linux, set the advertise address to the private IP
                            # of the machine.
                            args.advertise_address = self.find_ip_address_of_node()

                        else:
                            # Case Scenario: When advertise_address has no value in conf file
                            # and the OS is Mac, set the advertise address to 127.0.0.1.
                            args.advertise_address = IP_LOCALHOST
                else:
                    # Case Scenario: When advertise_address has a value in conf file and then
                    # use that regardless of the OS.
                    args.advertise_address = self.configs.saved_data.get("advertise_address")
            else:
                if not self.validate_hostname_ip(args.advertise_address):
                    has_errors = True
                    Output.print_and_log(Output.make_red("ERROR") + ": --advertise_address " +
                        "provided is not a valid address. Please try again with a valid IPV4, " +
                        "IPV6 or DNS.")

            args.advertise_address = IP_LOCALHOST if args.advertise_address == IP_ANY \
                else args.advertise_address

            if args.daemon is not None:
                if args.background is not None and \
                        self.parse_bool(args.daemon) != self.parse_bool(args.background):
                    Output.print_out(Output.make_red("ERROR") +
                        ": --daemon and --background flags " +
                        "are same. --daemon is depricated. Can't have different values for them.")
                    has_errors = True
                args.background = args.daemon

            if args.background is None:
                args.background = "True"

            cluster_member = self.configs.saved_data.get("cluster_member")
            join_ip = args.join if args.join else self.configs.saved_data.get("join")
            if join_ip:
                if not self.validate_hostname_ip(join_ip):
                    Output.log_error_and_exit(Output.make_red("ERROR") + ": --join" +
                        " provided is not a valid address. Please try again with a " +
                        "valid IPV4, IPV6 or DNS.")

                if not cluster_member:
                    # Node is starting for the first time.
                    Output.print_and_log("Fetching configs from join IP...")

                    # Check if tserver webserver at join_IP is reachable or not
                    # Also get the leader master(used to get the info of all tservers)
                    master_leader = self.get_current_master_leader_from_api(join_ip,
                                                                tserver_webserver_port)
                    join_ip = master_leader

                    # Get info on all tservers
                    master_leader_hostport = "{}:{}".format(get_url_from_ip(master_leader),
                                                            master_webserver_port)
                    tservers_info = dict(self.get_all_tserver_info(master_leader_hostport))

                    # Check if any existing node has the same IP as advertise address
                    for uuid, nodes in tservers_info.items():
                        for node in [urlparse("//" + node).hostname for node in list(nodes.keys())]:
                            if args.advertise_address == node:
                                Output.log_error_and_exit(Output.make_red("ERROR:") + " A node" +
                                    " is already running on {}, please ".format(join_ip) +
                                    "specify a valid address.")

                    is_placement_uuid_set = False

                    # Set placement UUID for the node according to it's properties(rr or primary)
                    if args.read_replica:
                        # When the 1st read replica node is started use a new uuid
                        if len(tservers_info) == 1:
                            is_placement_uuid_set = True
                            Output.log("Starting first read replica node. " +
                                        "Using {} as placement_uuid".format(
                                            self.configs.saved_data.get("placement_uuid")))
                        # When a read replica cluster exists use the existing placement UUID
                        else:
                            for uuid, nodes in tservers_info.items():
                                nodes_list = [urlparse("//" + node).hostname
                                              for node in list(nodes.keys())]
                                if master_leader not in nodes_list and len(nodes) != 0:
                                    self.configs.saved_data["placement_uuid"] = uuid
                                    Output.log("Using placement_uuid {} from ".format(uuid) +
                                                "existing read replica cluster.")
                                    is_placement_uuid_set = True
                    else:
                        # Use placement uuid set for the primary cluster when 1st node was started.
                        for uuid, nodes in tservers_info.items():
                            nodes_list = [urlparse("//" + node).hostname
                                          for node in list(nodes.keys())]
                            if master_leader in nodes_list:
                                self.configs.saved_data["placement_uuid"] = uuid
                                Output.log("Using placement_uuid {} from ".format(uuid) +
                                                "existing primary cluster.")
                                is_placement_uuid_set = True

                    # If placement UUID could not be set for some reason, throw an error
                    if not is_placement_uuid_set:
                        Output.log("Cannot find placement UUID for the node. " +
                                "Leader Master node: {}. ".format(master_leader) +
                                "Response from tablet-servers API: {}".format(
                                    str(tservers_info)))
                        Output.log_error_and_exit(Output.make_red("ERROR:") +
                                " Unable to start the node.")

                # Restart node as a part of an existing cluster with the join flag specified
                else:
                    Output.log("Restarting node as part of an existing cluster. " +
                                            "Using {} as placement_uuid".format(
                                            self.configs.saved_data.get("placement_uuid")))

            # If no --join is passed, check if its a first time start or its a restart
            else:
                if not cluster_member:
                    Output.log("Starting first primary node. Using {} as placement_uuid".format(
                                           self.configs.saved_data.get("placement_uuid")))
                # Restart node as a part of an existing cluster without the join flag specified
                else:
                    Output.log("Restarting node as part of an existing cluster. " +
                                            "Using {} as placement_uuid".format(
                                            self.configs.saved_data.get("placement_uuid")))

            self.find_security_nature_of_deployment(args)

            if (args.certs_dir is not None) and (not args.secure):
                # Case Sceneario: When certs_dir flag is passed without secure flag.
                has_errors = True
                Output.print_out(Output.make_red("ERROR") +
                    ": --certs_dir flag needs to be accompanied with the --secure flag.")

            if args.insecure:
                if join_ip and not cluster_member:
                    master_hostport = "{}:{}".format(get_url_from_ip(join_ip),
                                                     master_webserver_port)
                    if self.is_leader_master_secure(master_hostport):
                        # Case Scenario: When a User starts the 1st node in secure mode and tries
                        # to start the second node in insecure mode
                        has_errors = True
                        Output.print_out(Output.make_red("ERROR") + ": The node whose " +
                            "IP was provided in --join flag has SSL/TLS enabled. Cannot join a " +
                            "secure and an insecure node.")
            elif args.secure:
                if join_ip and not cluster_member:
                    master_hostport = "{}:{}".format(get_url_from_ip(join_ip),
                                                     master_webserver_port)
                    if not self.is_leader_master_secure(master_hostport):
                        # Case Scenario: When the user starts the 1st node in insecure mode and
                        # tries to start the second node in secure mode.
                        has_errors = True
                        Output.print_out(Output.make_red("ERROR") + ": The node whose " +
                            "IP was provided --join flag does not have SSL/TLS enabled. Cannot " +
                            "join a secure and an insecure node.")

                if not has_errors:
                    self.validate_security_configs(args)

                    args.ysql_enable_auth="true"
                    args.use_cassandra_authentication="true"

                    self.configs.saved_data["ca_cert_file_path"] = os.path.join(args.certs_dir,
                                                                                    "ca.crt")
            args.background = self.parse_bool(args.background)
            if args.ui is not None:
                args.ui = self.parse_bool(args.ui)
            else:
                args.ui = self.configs.saved_data.get("ui")

            if args.backup_daemon is not None:
                args.backup_daemon = self.parse_bool(args.backup_daemon)
            else:
                args.backup_daemon = self.configs.saved_data.get("backup_daemon")

            if not has_errors:
                self.configs.temp_data["ui_port_available"] = self.is_port_available(
                    args.advertise_address, self.configs.saved_data.get("yugabyted_ui_port"))

            if args.callhome is not None:
                args.callhome = self.parse_bool(args.callhome)
            elif os.environ.get("YB_DISABLE_CALLHOME") is not None:
                args.callhome = os.environ.get("YB_DISABLE_CALLHOME") not in TRUE_CHOICES
            else:
                args.callhome = DEFAULT_CALLHOME

            # Set authentication flags same as provided in the command-line flags.
            if args.ysql_enable_auth:
                args.ysql_enable_auth = self.parse_bool(args.ysql_enable_auth)

            if args.use_cassandra_authentication:
                args.use_cassandra_authentication = self.parse_bool(args.use_cassandra_authentication)

            # Set authentication flags to True, if it is first-run and
            # have required environment variables to enforce the authentication.
            if not self.is_yb_initialized():
                if self.setup_env_init.is_exists('YSQL_PASSWORD'):
                    args.ysql_enable_auth = True

                # Add use_cassandra_authentication flag to enforce authentication for YCQL
                if self.setup_env_init.is_exists('YCQL_USER') or \
                        self.setup_env_init.is_exists('YCQL_PASSWORD'):
                    args.use_cassandra_authentication = True

            self.configs.temp_data["background"] = args.background
            self.configs.saved_data["ui"] = args.ui
            self.configs.saved_data["backup_daemon"] = args.backup_daemon
            self.configs.temp_data["initial_scripts_dir"] = args.initial_scripts_dir
            self.configs.temp_data[
                "enhance_time_sync_via_clockbound"] = args.enhance_time_sync_via_clockbound

        if has_errors:
            sys.exit(1)

        parent_flags = ["log_dir", "data_dir"]
        # Override configs and defaults with user specified variables
        for k, v in get_kv(args.__dict__):
            if (v is not None and k not in parent_flags and k in self.configs.saved_data
                    and v != self.configs.saved_data.get(k)):
                self.configs.saved_data[k] = v

        self.configs.save_configs()

    def parse_bool(self, config):
        return config in TRUE_CHOICES

    def run(self):
        # Parent subparser for common args
        common_parser = argparse.ArgumentParser(add_help=False)
        # TODO: Refactor data_dir to be a list for multi-node. How should the config file and
        # data dir be set for local mulit-node setups? Note: daemon mode may be affected.
        common_parser.add_argument(
            "--data_dir", help=argparse.SUPPRESS)
        common_parser.add_argument(
            "--base_dir", help="Directory under which {0} will store data, conf and logs. It defaults to `~/var`. On nodes started with a non-default `--base_dir`, this flag must be specified on every command so that {0} can identify the correct node".format(
                SCRIPT_NAME), metavar="")
        common_parser.add_argument(
            "--log_dir", help=argparse.SUPPRESS)

        start_msg = "To start YugabyteDB cluster, run '{}'.\n\n".format(
            Output.make_green("{} start".format(SCRIPT_NAME)))
        start_msg += "Find more information at: {}".format(Output.make_underline(YUGABYTED_LINK))
        parser = PrettyArgParser(description=start_msg)
        all_parsers = {"default": parser}

        # Top Level Commands: start, stop, destroy, status, version, collect_logs
        subparsers = parser.add_subparsers(dest="parser", metavar="")
        subparsers.required = True
        for cmd, description in (
                ("start", "Start YugabyteDB cluster."),
                ("stop", "Stop running YugabyteDB cluster."),
                ("destroy", "Destroy YugabyteDB cluster and remove data."),
                ("backup", "Back up a database."),
                ("restore", "Restore a database."),
                ("status", "Print status of YugabyteDB cluster."),
                ("version", "Release version of YugabyteDB cluster."),
                ("finalize_upgrade", "Finalize the upgrade process for the YugabyteDB cluster."),
                ("collect_logs", "Collect and package logs for troubleshooting.")):
            example = ""
            if EXAMPLE.get(cmd):
                example = Output.make_yellow("Examples") + ": \n" + EXAMPLE[cmd]
            subparser = subparsers.add_parser(cmd, description=example,
                                                help=description, parents=[common_parser])
            subparser.epilog = EPILOG_SPECIFIC[cmd] + EPILOG_COMMON
            func = getattr(self, cmd, None)
            subparser.set_defaults(func=func)
            all_parsers[cmd] = subparser

        # Top Level command: connect
        connect = subparsers.add_parser("connect",
                    help="Connect to YugabyteDB cluster through the CLI.")
        all_parsers["connect"] = connect

        # Sub-commands for top level `connect` command: ysql, ycql
        connect_subparser = connect.add_subparsers(dest="parser", metavar="")
        connect_subparser.required = True
        for api in YUGABYTE_API_CLIENT_PROGRAMS:
            cur_parser = connect_subparser.add_parser(
                api, help="Use {} through the CLI.".format(api.upper()), parents=[common_parser])
            func = getattr(self, "connect_{}".format(api), None)
            cur_parser.set_defaults(func=func)
            all_parsers[api] = cur_parser

        # Flags for sub command: ysql
        for cmd in ("ysql",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--username", help="YSQL username to connect to" \
                " the YugabyteDB", metavar="")
            cur_parser.add_argument(
                "--password", help="The password for YSQL username to connect" \
                " to the YugabyteDB", metavar="")
            cur_parser.add_argument(
                "--database", help="Name of the YSQL database to connect" \
                " to the YugabyteDB", metavar="")

        # Flags for sub command: ycql
        for cmd in ("ycql",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--username", help="YCQL username to connect to" \
                " the YugabyteDB", metavar="")
            cur_parser.add_argument(
                "--password", help="The password for YCQL username to connect" \
                " to the YugabyteDB", metavar="")
            cur_parser.add_argument(
                "--keyspace", help="Name of the YCQL keyspace to connect" \
                " to the YugabyteDB", metavar="")

        # Top Level command: demo
        demo_parser = subparsers.add_parser("demo", help="Load and interact with preset demo data.")
        all_parsers["demo"] = demo_parser

        # Sub-commands for top level `demo` command: connect, destroy
        demo_subparsers = demo_parser.add_subparsers(dest="parser", metavar="")
        demo_subparsers.required = True
        for cmd, description in (
                ("connect", "Connect to the demo database."),
                ("destroy", "Destroy the demo database.")):
            subparser = demo_subparsers.add_parser(cmd, help=description, parents=[common_parser])
            parser_name = cmd + "_demo"
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Top Level command: cert
        cert_example = Output.make_yellow("Examples") + ": \n" + EXAMPLE["cert"]
        cert_parser = subparsers.add_parser("cert", description=cert_example,
                                                help="Generate SSL certificates")
        all_parsers["cert"] = cert_parser
        cert_subparsers = cert_parser.add_subparsers(dest="parser", metavar="")
        cert_subparsers.required = True
        for cmd, description in (
                ("generate_server_certs", "Generate node server certificates."),):
            subparser = cert_subparsers.add_parser(cmd, help=description, parents=[common_parser])
            parser_name = "cert_" + cmd
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Sub-commands for top level `cert` command: generate_server_certs
        for cmd in ("cert_generate_server_certs",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--hostnames", help="Hostnames of the nodes to be added in the cluster. " +
                "Mandatory flag.", metavar="")

        # Flags for top level `collect_logs` command
        # Docker: Redirect the logs.tar.gz to stdout
        for cmd in ("collect_logs",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--stdout", help="Redirect the logs.tar.gz file's content to stdout. Ex: "+
                "docker exec <container-id> bin/yugabyted collect_logs --stdout > yugabyted.tar.gz",
                action="store_true", default=False)
            cur_parser.add_argument(
                "--collect_at_dir", help="Directory under which {} will store logs.tar.gz file"
                .format(SCRIPT_NAME),metavar="")

        # Top Level command: configure
        configure_example = Output.make_yellow("Examples") + ": \n" + EXAMPLE["configure"]
        configure_parser = subparsers.add_parser("configure", description=configure_example,
                                                help="Configure data placement," +
                                                " toggle encryption at rest or run" +
                                                " point-in-time recovery operations on" +
                                                " the cluster.", parents=[common_parser])

        # Sub-commands for top level `configure` command: data_placement,
        # encrypt_at_rest, admin_operation, point_in_time_recovery
        all_parsers["configure"] = configure_parser
        configure_subparsers = configure_parser.add_subparsers(dest="parser", metavar="")
        configure_subparsers.required = True
        for cmd, description in (
        ("data_placement", "Configure multi-zone/multi-region cluster."),
        ("encrypt_at_rest", "Enable or disable encryption at rest."),
        ("admin_operation", "Run yb-admin command on the YugabyteDB cluster."),
        ("point_in_time_recovery", "Configure point-in-time recovery for "
            "a database or keyspace."),):
            parser_name = "configure_" + cmd
            example = ""
            if EXAMPLE.get(parser_name):
                example = Output.make_yellow("Examples") + ": \n" + EXAMPLE[parser_name]
            subparser = configure_subparsers.add_parser(cmd, help=description, description=example,
                                parents=[common_parser])
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Flags for sub-command: data_placement
        # Flags for muti-zone/multi-region configuration.
        for cmd in ("configure_data_placement",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--fault_tolerance", help="No more than 1 node in the same " +
                "fault tolerance can be a master",
                metavar="")
            cur_parser.add_argument(
                "--constraint_value", help="Data placement constriant to be applied " +
                "on the YugabyteDB Cluster",
                metavar="")
            cur_parser.add_argument(
                "--rf", help="Set the replication factor for each tablet",
                metavar="")

        # Flags for sub-command: encrypt_at_rest
        for cmd in ("configure_encrypt_at_rest",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--enable", help="Enable encryption at rest for the cluster. No need to set a " +
                "value for the flag. Use --enable or --disable flag to toggle encryption " +
                "features on a YugabyteDB cluster.",
                action="store_true")
            cur_parser.add_argument(
                "--disable", help="Disable encryption at rest for the cluster. No need to set a " +
                "value for the flag. Use --enable or --disable flag to toggle encryption " +
                "features on a YugabyteDB cluster.",
                action="store_true")

        # Flags for sub-command: admin_operation
        for cmd in ("configure_admin_operation",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--command", help="specify the yb-admin command to be executed " +
                                        "on the YugabyteDB Cluster",
                metavar="")
            cur_parser.add_argument(
                "--master_addresses", help="specify the comma seperated values of current " +
                "masters of the cluster.",
                metavar=""
            )

        # Flags for sub-command: point_in_time_recovery
        for cmd in ("configure_point_in_time_recovery",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--enable", help="Enable point-in-time recovery for a database or keyspace.",
                action="store_true", default=None
                )
            cur_parser.add_argument(
                "--retention", help="Specify the retention period for snapshots in days (from the time they were created), after which they will be automatically deleted.",
                metavar=""
                )
            cur_parser.add_argument(
                "--disable", help="Disable point-in-time recovery for a database or keyspace.",
                action="store_true", default=None
                )
            cur_parser.add_argument(
                "--database", help="Name of the YSQL database for which " + \
                "point-in-time recovery is being configured.",
                metavar=""
                )
            cur_parser.add_argument(
                "--keyspace", help="Name of the YCQL keyspace for which " + \
                "point-in-time recovery is being configured.",
                metavar=""
                )
            cur_parser.add_argument(
                "--status", help="Display point-in-time recovery status for a " + \
                "YugabyteDB cluster.", action="store_true", default=None
                )

        # Top Level command: configure_read_replica
        configure_read_replica_example = Output.make_yellow("Examples") + ": \n" + \
                                                EXAMPLE["configure_read_replica"]
        configure_read_replica_parser = subparsers.add_parser("configure_read_replica",
                                                description=configure_read_replica_example,
                                                help="Configure/Modify/" +
                                                "Delete a read replica cluster.",
                                                parents=[common_parser])

        # Sub-commands for top level `configure_read_replica` command: new, modify, delete
        all_parsers["configure_read_replica"] = configure_read_replica_parser
        configure_read_replica_subparsers = configure_read_replica_parser.add_subparsers(
                                                                    dest="parser", metavar="")
        configure_read_replica_subparsers.required = True
        for cmd, description in (
                ("new", "Configure a new read replica cluster."),
                ("modify", "Modify an existing new read replica cluster."),
                ("delete", "Delete an existing new read replica cluster.")):
            parser_name = "configure_read_replica_" + cmd
            example = ""
            if EXAMPLE.get(parser_name):
                example = Output.make_yellow("Examples") + ": \n" + EXAMPLE[parser_name]
            subparser = configure_read_replica_subparsers.add_parser(cmd, help=description,
                                description=example, parents=[common_parser])
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Flags for sub-command: new
        for cmd in ("configure_read_replica_new",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--rf", help="Replication factor for read replica cluster.",
                metavar="")
            cur_parser.add_argument(
                "--data_placement_constraint", help="Placement policy for the read replica " +
                "cluster.", metavar="")

        # Flags for sub-command: modify
        for cmd in ("configure_read_replica_modify",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--rf", help="Replication factor for read replica cluster.",
                metavar="")
            cur_parser.add_argument(
                "--data_placement_constraint", help="Placement policy for the read replica " +
                "cluster.", metavar="")

        # Top level command: backup
        for cmd in ("backup",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--database", metavar="",
                help="YSQL Database to be backed up to cloud storage.")
            cur_parser.add_argument(
                "--keyspace", metavar="",
                help="YCQL Keyspace to be backed up to cloud storage.")
            cur_parser.add_argument(
                "--cloud_storage_uri", metavar="",
                help="Cloud location to store the backup data.")
            cur_parser.add_argument(
                "--status",
                help="Check the status of the backup task.",
                action="store_true", default=None
            )

        # Top level command: restore
        for cmd in ("restore",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--database", metavar="",
                help="YSQL Database to be backed up to cloud storage.")
            cur_parser.add_argument(
                "--keyspace", metavar="",
                help="YCQL Keyspace to be backed up to cloud storage.")
            cur_parser.add_argument(
                "--cloud_storage_uri", metavar="",
                help="Cloud location to store the backup data.")
            cur_parser.add_argument(
                "--status",
                help="Check the status of the restore task.",
                action="store_true", default=None
            )
            cur_parser.add_argument(
                "--recover_to_point_in_time", help="Restore to the specified point" + \
                "-in-time with timestamp enclosed within single quotes.",
                metavar=""
            )

        # Top level command: finalize_upgrade
        for cmd in ("finalize_upgrade",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--upgrade_ysql_timeout",
                help="Custom timeout for the YSQL upgrade in milliseconds.",
                metavar=""
            )

        # Top level command: upgrade
        upgrade_parser = subparsers.add_parser("upgrade",
            description="Upgrade YugabyteDB processes in this node.",
            help="Upgrade YugabyteDB processes in this node.",
            parents=[common_parser])

        # Sub-commands for top level `xcluster` command
        all_parsers["upgrade"] = upgrade_parser
        upgrade_subparsers = upgrade_parser.add_subparsers(dest="parser", metavar="")
        upgrade_subparsers.required = True
        for cmd, description in (
                ("ysql_catalog", "Upgrade the ysql catalog metadata to the new version."),
                ("finalize_new_version", "Finalize the YugabyteDB upgrade to the new version."),
                ("check_version_compatibility",
                    "Check the version compatibility of the YugabyteDB."),
        ):
            parser_name = "upgrade_" + cmd
            example  = ""
            if EXAMPLE.get(parser_name):
                example = Output.make_yellow("Examples") + ": \n" + EXAMPLE[parser_name]
            subparser = upgrade_subparsers.add_parser(cmd, help=description,
                                description=example, parents=[common_parser])
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Flags for sub-command: ysql_catalog
        for cmd in ("upgrade_ysql_catalog",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--timeout", help="Custom timeout for the YSQL catalog upgrade in milliseconds.",
                metavar=""
            )

        for cmd in ("upgrade_check_version_compatibility",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--timeout", help="Custom timeout for the YugabyteDB node upgrade " +
                    "compatibility check in milliseconds.", metavar="",)

        # Flags for sub-command: server
        for cmd in ("upgrade_finalize_new_version",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--timeout", help="Custom timeout for the YugabyteDB node upgrade in milliseconds.",
                metavar=""
            )

        # Top Level command: xcluster
        xcluster_example = Output.make_yellow("Examples") + ": \n" + EXAMPLE["xcluster"]
        xcluster_parser = subparsers.add_parser("xcluster",
                                    description=xcluster_example,
                                    help="Operations for xcluster replication between 2 clusters.",
                                    parents=[common_parser])

        # Sub-commands for top level `xcluster` command
        all_parsers["xcluster"] = xcluster_parser
        xcluster_subparsers = xcluster_parser.add_subparsers(dest="parser", metavar="")
        xcluster_subparsers.required = True
        for cmd, description in (
                ("create_checkpoint", "Initialise a xcluster replication. " +
                                        "Needs to be run from source cluster."),
                ("add_to_checkpoint", "Checkpoint database which is to be added to an existing " +
                                        "xcluster replication. " +
                                        "Needs to be run from source cluster."),
                ("set_up", "Setup xcluster replication. " +
                                        "Needs to be run from source cluster."),
                ("add_to_replication", "Add database to xcluster replication. " +
                                        "Needs to be run from source cluster."),
                ("status", "Displays the status of any replications running " +
                                        "from or to this cluster." +
                                        "Can be run from either source or target cluster."),
                ("delete_replication", "Deletes the specified replication. " +
                                        "Needs to run from source cluster"),
                ("remove_database_from_replication", "Deletes the specified database " +
                                        "from the replication." +
                                        " Needs to run from source cluster"),):
            parser_name = "xcluster_" + cmd
            example = ""
            if EXAMPLE.get(parser_name):
                example = Output.make_yellow("Examples") + ": \n" + EXAMPLE[parser_name]
            subparser = xcluster_subparsers.add_parser(cmd, help=description,
                                description=example, parents=[common_parser])
            func = getattr(self, parser_name, None)
            subparser.set_defaults(func=func)
            all_parsers[parser_name] = subparser

        # Flags for sub-command: create_checkpoint
        for cmd in ("xcluster_create_checkpoint",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--replication_id", help="Unique string to assign to a replication.",
                metavar="")
            cur_parser.add_argument(
                "--databases", help="Name of database to be replicated.",
                metavar="")
            cur_parser.add_argument(
                "--automatic_mode",
                help="Setup xCluster in Automatic mode.",
                action="store_true")

        # Flags for sub-command: create_checkpoint
        for cmd in ("xcluster_add_to_checkpoint",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--replication_id", help="Replication id of the xcluster replication to which " +
                "database is to be added.",
                metavar="")
            cur_parser.add_argument(
                "--databases", help="Name of databases to be added.",
                metavar="")

        # Flags for sub-command: set_up
        for cmd in ("xcluster_set_up",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--target_address",
                help="IP of any target cluster node.",
                metavar="")
            cur_parser.add_argument(
                "--replication_id",
                help="Unique string to assigned to a replication during create_checkpoint.",
                metavar="")
            cur_parser.add_argument(
                "--bootstrap_done",
                help="Use this flag to indicate bootstrapping has been completed. Mandatory flag",
                action="store_true")

        for cmd in ("xcluster_add_to_replication",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--target_address",
                help="IP of any target cluster node.",
                metavar="")
            cur_parser.add_argument(
                "--replication_id",
                help="Replication id of the xcluster replication to which database is to be added.",
                metavar="")
            cur_parser.add_argument(
                "--databases", help="Name of databases to be added.",
                metavar="")
            cur_parser.add_argument(
                "--bootstrap_done",
                help="Use this flag to indicate bootstrapping has been completed. Mandatory flag",
                action="store_true")

        # Flags for sub-command: set_up
        for cmd in ("xcluster_status",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--replication_id",
                help="Replication id of the xcluster replication whose status is to be displayed.",
                metavar="")

        # Flags for sub-command: delete_replication
        for cmd in ("xcluster_delete_replication",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--target_address",
                help="IP of any target cluster node.",
                metavar="")
            cur_parser.add_argument(
                "--replication_id",
                help="Unique string to assigned to a replication during create_checkpoint.",
                metavar="")

        # Flags for sub-command: remove_database_from_replication
        for cmd in ("xcluster_remove_database_from_replication",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--target_address",
                help="IP of any target cluster node.",
                metavar="")
            cur_parser.add_argument(
                "--replication_id",
                help="Unique string to assigned to a replication during create_checkpoint.",
                metavar="")
            cur_parser.add_argument(
                "--databases", help="Name of databases to be removed.",
                metavar="")

        # Args for stop command
        for cmd in ("stop",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--upgrade", choices=BOOL_CHOICES,
                help="Stop the yugabyted node for version Upgrade.",
                metavar="")

        # Commands that can alter configuration file.
        for cmd in ("start",):
            cur_parser = all_parsers[cmd]
            cur_parser.add_argument(
                "--ycql_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--ysql_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--master_rpc_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--tserver_rpc_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--master_webserver_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--tserver_webserver_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--webserver_port", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--listen", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--advertise_address",
                help="IP address or local hostname on which {} will listen.".format(SCRIPT_NAME),
                metavar="")
            cur_parser.add_argument(
                "--join", help="IP address to which this process will join",
                metavar="")
            cur_parser.add_argument(
                "--read_replica", help="Use this flag to start a read replica node.",
                action="store_true", default=None)
            cur_parser.add_argument(
                "--secure", help="Start a YugabyteDB cluster in secure mode with encryption in " +
                "transit and password authentication enabled. No need to set a value for the " +
                "flag. Use --secure or --insecure flag to toggle security features on a " +
                "YugabyteDB cluster.",
                action="store_true", default=None)
            cur_parser.add_argument(
                "--insecure", help="Start a YugabyteDB cluster in an insecure mode without " +
                "encryption in transit and password authentication enabled. For non-production " +
                "use only, not to be used without firewalls blocking the internet traffic. No " +
                "need to set a value for the flag. Use --secure or --insecure flag to toggle " +
                "security features on a YugabyteDB cluster.", action="store_true", default=None)
            cur_parser.add_argument(
                "--certs_dir", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--daemon", choices=BOOL_CHOICES, help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--background", choices=BOOL_CHOICES, help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--callhome", choices=BOOL_CHOICES,
                help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--ui", choices=BOOL_CHOICES, metavar="",
                help="Toggle enabling or disabling webserver UI. Default true.")
            cur_parser.add_argument(
                "--backup_daemon", choices=BOOL_CHOICES, metavar="",
                help="Toggle enabling or disabling YBC server. Default False.")
            cur_parser.add_argument(
                "--ysql_enable_auth", choices=BOOL_CHOICES,
                help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--use_cassandra_authentication", choices=BOOL_CHOICES,
                help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--initial_scripts_dir", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--cloud_location", metavar="",
                help="Cloud location of the node in form of cloudprovider.region.zone")
            cur_parser.add_argument(
                "--fault_tolerance", metavar="",
                help="Determines the type of deployment of the cluster. Default is None.")
            cur_parser.add_argument(
            "--config", help="{} user configuration file path".format(
                              SCRIPT_NAME), metavar="")
            cur_parser.add_argument(
                "--enhance_time_sync_via_clockbound",
                help="Enable clock bound for the node. Default value is False.",
                action="store_true", default=False)

            # Hidden commands for development/advanced users
            cur_parser.add_argument(
                "--polling_interval", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--master_flags", help=argparse.SUPPRESS)
            cur_parser.add_argument(
                "--tserver_flags", help=argparse.SUPPRESS)


        if not sys.argv[1:]:
            parser.print_help()
            return

        args = parser.parse_args()
        self.validate_and_set_parent_configs(args)
        self.diagnostics = Diagnostics(self.configs)
        # Yugabyted command currently being processed is required for
        # generating the status string.
        self.configs.temp_data["yugabyted_cmd"] = args.parser

        log_dir = self.configs.saved_data.get("log_dir")
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)

        Output.log_dir = log_dir
        logging.basicConfig(
            level=logging.DEBUG, filemode="a",
            filename=os.path.join(log_dir, "{}.log".format(SCRIPT_NAME)),
            format="[%(filename)s " + args.parser + "] %(asctime)s %(levelname)s: %(message)s")

        Output.log("Running yugabyted command: '{}'".format(' '.join(sys.argv)))

        Output.log("cmd = {} using config file: {}".format(args.parser, self.conf_file))

        if args.parser == "start" and args.config:
            Output.log("Updating yugabyted config file with"
                    " user-specified configs located at: {}".format(args.config))

        # Initialize the script path of openssl_proxy.sh
        OpenSSLProxy.init()

        # Initialize the binary path of ybadmin
        YBAdminProxy.init()

        # Initialize the binary path of yb-ts-cli
        YBTableServerCLIProxy.init()

        # Initialize the binary path of pg_upgrade
        PGUpgradeProxy.init()

        self.validate_and_set_configs(args)

        # Initialize the binary path of ybadmin
        # TODO(Sanket): Clean up and refactor this file
        YBAdminProxy.set_certs_dir(
            self.configs.saved_data.get("master_flags"), self.configs.saved_data.get("secure"),
            self.configs.saved_data.get("certs_dir"), self.find_hostname_of_node())

        YBTableServerCLIProxy.set_certs_dir(self.configs.saved_data.get("master_flags"), \
            self.configs.saved_data.get("secure"), self.configs.saved_data.get("certs_dir"))

        try:
            args.func()
        except Exception as e:
            Output.print_out(
                "{} crashed. For troubleshooting, contact us on {} or check our FAQ at {}".format(
                    SCRIPT_NAME, Output.make_underline(SLACK_LINK),
                    Output.make_underline(HELP_LINK)))
            Output.log_error_and_exit(traceback.format_exc())

    def advertise_ip(self):
        bind_ip = self.configs.saved_data.get("advertise_address")
        return bind_ip if bind_ip != IP_ANY else IP_LOCALHOST

    def master_port(self):
       self.configs.saved_data.get("master_rpc_port")

    def first_install_init_auth(self):
        if self.get_failed_node_processes():
            Output.log_error_and_exit(
                "{} is not running.".format(SCRIPT_NAME))

        if self.setup_env_init.is_exists('YSQL_USER') or \
                self.setup_env_init.is_exists('YSQL_PASSWORD') or \
                self.setup_env_init.is_exists('YSQL_DB'):
            ysql_proxy = YsqlProxy(ip=self.advertise_ip(),
                            port=self.configs.saved_data.get("ysql_port"),
                            get_default_credentials=True)
            if retry_op(ysql_proxy.is_ysql_up):
                Output.log("Setting up custom credentials for YSQL...")
                self.setup_env_init.setup_ysql_credentials(ysql_proxy)

        if self.setup_env_init.is_exists('YCQL_USER') or \
                self.setup_env_init.is_exists('YCQL_PASSWORD') or \
                self.setup_env_init.is_exists('YCQL_KEYSPACE'):
            ycql_proxy = YcqlProxy(ip=self.advertise_ip(),
                            port=self.configs.saved_data.get("ycql_port"),
                            get_default_credentials=True)
            if retry_op(ycql_proxy.is_ycql_up):
                Output.log("Setting up custom credentials for YCQL...")
                self.setup_env_init.setup_ycql_credentials(ycql_proxy)

        if self.configs.temp_data.get("initial_scripts_dir"):
            init_scripts = os.path.abspath(self.configs.temp_data.get("initial_scripts_dir"))

            if os.path.exists(init_scripts):
                Output.log("Initialization scripts from the {} directory".format(init_scripts))

                sql_files = sorted([sql_file for sql_file in os.listdir(init_scripts) if (
                                sql_file.endswith('.sql'))])
                cql_files = sorted([cql_file for cql_file in os.listdir(init_scripts) if (
                                cql_file.endswith('.cql'))])

                ysql_proxy = YsqlProxy(ip=self.advertise_ip(),
                                port=self.configs.saved_data.get("ysql_port"))
                if sql_files and retry_op(ysql_proxy.is_ysql_up):
                    self.load_init_scripts(ysql_proxy, init_scripts, sql_files)

                ycql_proxy = YcqlProxy(ip=self.advertise_ip(),
                                port=self.configs.saved_data.get("ycql_port"))
                if cql_files and retry_op(ycql_proxy.is_ycql_up):
                    self.load_init_scripts(ycql_proxy, init_scripts, cql_files)

    def load_init_scripts(self, proxy_class, init_scripts_dir, files):
        files_path = []
        for name in files:
            files_path.append(os.path.join(init_scripts_dir, name))

        proxy_class.load_files(files_path)

class Configs(object):
    def __init__(self, config_file, base_dir):
        self.saved_data = {
            "data_dir": os.path.join(base_dir, "data"),
            "additional_data_dir": "",
            "log_dir": os.path.join(base_dir, "logs"),
            "gen_certs_dir": os.path.join(base_dir, "generated_certs"),
            "master_rpc_port": DEFAULT_MASTER_RPC_PORT,
            "tserver_rpc_port": DEFAULT_TSERVER_RPC_PORT,
            "master_webserver_port": DEFAULT_MASTER_WEBSERVER_PORT,
            "tserver_webserver_port": DEFAULT_TSERVER_WEBSERVER_PORT,
            "ysql_port": DEFAULT_YSQL_PORT,
            "ycql_port": DEFAULT_YCQL_PORT,
            "ysql_metric_port": DEFAULT_YSQL_METRIC_PORT,
            "ycql_metric_port": DEFAULT_YCQL_METRIC_PORT,
            "advertise_address": "",
            "webserver_port": DEFAULT_WEBSERVER_PORT,
            "yugabyted_ui_port": DEFAULT_YUGABYTED_UI_PORT,
            "universe_uuid": str(uuid.uuid4()),
            "node_uuid": str(uuid.uuid4()),
            "tserver_uuid": str(uuid.uuid4()).replace("-", ""),
            "master_uuid": str(uuid.uuid4()).replace("-", ""),
            "placement_uuid": str(uuid.uuid4()),
            "polling_interval": "5",
            "callhome": DEFAULT_CALLHOME,
            "master_flags": "",
            "tserver_flags": "",
            "join": "",
            "ysql_enable_auth": False,
            "use_cassandra_authentication": False,
            "cloud_provider": DEFAULT_CLOUD_PROVIDER,
            "cloud_region": DEFAULT_CLOUD_REGION,
            "cloud_zone": DEFAULT_CLOUD_ZONE,
            "fault_tolerance": DEFAULT_START_FAULT_TOLERANCE,
            "secure": False,
            "insecure": True,
            "certs_dir": os.path.join(base_dir, "certs"),
            "ca_cert_file_path": "",
            "database_password": None,
            "current_masters": "",
            "ui": True,
            "backup_daemon": False,
            "dns_enabled": False,
            "read_replica": False,
            "cluster_member": False,
            "upgrade_in_progress": False,
            "old_release_cmd": "",
            "catalog_upgrade_completed": False,
            "created_on": 0,
        }
        # Used to store data specific to certain functions that we don't want to save.
        self.temp_data = {
            "demo_db": DEFAULT_DEMO_DATABASE,
            "background": True,
            "initial_scripts_dir": "",
            "collect_logs_stdout": False,
            "constraint_value": "",
            "replication_factor":"",
            "fault_tolerance": DEFAULT_FAULT_TOLERANCE,
            "hostnames": "",
            "enable_encrypt_at_rest": False,
            "disable_encrypt_at_rest": False,
            "admin_command": "",
            "admin_operation_master_addresses":"",
            "rr_data_placement_constraint": "",
            "rr_replication_factor": "",
            "collect_at_dir": os.path.join(os.path.expanduser("~"), "yugabyte_collected_logs"),
            "enable_pitr": False,
            "disable_pitr": False,
            "retention_period": DEAFULT_RETENTION_PERIOD,
            "restore_time": None,
            "ui_port_available": True,
            "database_name_backup_restore": None,
            "keyspace_name_backup_restore": None,
            "ybc_cloud_type": "",
            "ybc_cloud_storage_bucket": "",
            "ybc_cloud_storage_dir": "",
            "ybc_status": False,
            "upgrade_ysql_timeout": DEFAULT_UPGRADE_YSQL_TIMEOUT,
            "xcluster_replication_id": "",
            "xcluster_databases": "",
            "xcluster_target_addresses": "",
            "xcluster_bootstrap_done": "",
            "enhance_time_sync_via_clockbound": False,
            "is_clockbound_configured": False,
            "upgrade_timeout": None,
        }
        self.config_file = config_file

    # Saves current configs to config file.
    def save_configs(self):
        if os.path.exists(self.config_file):
            with open(self.config_file, "w+") as f:
                json.dump(self.saved_data, f, indent=4)

    # Custom parser for reading config file.
    @staticmethod
    def parse_config_file(config_file, base_dir):
        configs = Configs(config_file, base_dir)
        if os.path.isfile(config_file):
            try:
                with open(config_file) as f:
                   configs.saved_data.update(json.load(f))
            except ValueError as e:
                Output.log_error_and_exit(
                    "Failed to read config file {}: {}".format(config_file, str(e)))

        configs.saved_data = Configs.expand_path_variables(configs.saved_data)

        return configs

    # Custom parser for reading user config file.
    @staticmethod
    def parse_user_config_file(user_conf_file):
        user_configs = {}
        if not os.path.exists(user_conf_file):
            Output.log_error_and_exit("User config file {} does not exist.".format(user_conf_file))
        try:
            with open(user_conf_file, 'r') as f:
                content = f.read()
                # If the file is empty or contains only whitespace,
                # user_configs remains an empty dictionary
                if content.strip():
                    user_configs = json.loads(content)
        except Exception as e:
            Output.log_error_and_exit("Failed to read user config file {}: {}".format(
                                        user_conf_file, str(e)))

        user_configs = Configs.expand_path_variables(user_configs)

        return user_configs

    # Expand path variables in config files
    @staticmethod
    def expand_path_variables(configs):
        paths = ["log_dir", "certs_dir", "gen_certs_dir", "ca_cert_file_path"]
        multi_paths = ["data_dir"]
        for key, value in configs.items():
            if value is not None:
                if key in multi_paths:
                    configs[key] = ','.join([
                        os.path.abspath(os.path.realpath(
                            os.path.expanduser(os.path.expandvars(path))) \
                            if "$" in path else \
                            os.path.abspath(os.path.realpath(os.path.expanduser(path)))
                        ) for path in value.split(',')
                    ])
                elif key in paths:
                    configs[key] = (
                        os.path.abspath(os.path.realpath(
                            os.path.expanduser(os.path.expandvars(value))))
                        if "$" in value else
                        os.path.abspath(os.path.realpath(os.path.expanduser(value)))
                    )

        return configs

    @staticmethod
    def get_brew_config():
        # hack alert: we are using the cellar dir name to identify a brew install
        if ("darwin" == sys.platform and
            os.path.realpath(sys.argv[0]).lower().find('cellar') >= 0 and
            os.path.exists(BREW_CONF_FILE)):
            return BREW_CONF_FILE

        return None

    # Returns information about demo databases.
    @staticmethod
    def get_demo_info():
        return {
            "retail": {
                "files": ("schema.sql", "products.sql", "users.sql", "reviews.sql", "orders.sql"),
                "output": "    Database: yb_demo_retail\n"
                        "    |_ users\n"
                        "    |_ products\n"
                        "    |_ orders\n"
                        "    |_ reviews\n\n",
                "examples": "# JOINS (find user details of orders):\n"
                        "    %s users.id, users.name, users.email, orders.id, orders.total\n"
                        "        %s orders %s users %s orders.user_id=users.id\n"
                        "        %s 10;\n\n" % tuple([Output.make_cyan(kw) for kw in (
                            "SELECT", "FROM", "INNER JOIN", "ON", "LIMIT")])
            },
            "northwind": {
                "files": ("northwind_ddl.sql", "northwind_data.sql"),
                "output": "",
                "examples": "# JOINS (find customer details for orders):\n"
                        "   %s c.customer_id, c.company_name, o.order_id, o.order_date\n"
                        "       %s orders o %s customers c %s o.customer_id=c.customer_id\n"
                        "       %s 10;\n\n" % tuple([Output.make_cyan(kw) for kw in (
                            "SELECT", "FROM", "INNER JOIN", "ON", "LIMIT")])
            },
            "club": {
                "files": ("clubdata_ddl.sql", "clubdata_data.sql"),
                "output": "",
                "examples": ""
            },
            "sports": {
                "files": (
                    "sportsdb_tables.sql", "sportsdb_fks.sql",
                    "sportsdb_indexes.sql", "sportsdb_inserts.sql"),
                "output": "",
                "examples": ""
            }
        }


class ProcessManager(object):
    def __init__(self, name, cmd, log_dir, data_dir, process_log_dir=""):
        self.name = name
        self.cmd = cmd
        self.log_dir = log_dir
        self.data_dir = data_dir
        self.pidfile = os.path.join(self.data_dir, "{}.pid".format(name))
        self.process = None
        self.start_time = None
        self.process_log_dir = process_log_dir

    # Start process. Creates pidfile and corresponding output logs.
    def start(self):
        Output.log("About to start {} with cmd {}".format(self.name, " ".join(self.cmd)))
        out_log = os.path.join(self.log_dir, "{}.out".format(self.name))
        err_log = os.path.join(self.log_dir, "{}.err".format(self.name))
        with open(out_log, "a") as out_log, open(err_log, "a") as err_log:
            self.process = subprocess.Popen(
                self.cmd, stdout=out_log, stderr=err_log, preexec_fn=self.set_rlimits)
            self.start_time = time.time()
        self.write_pid(self.process.pid)

        # Add symlink to the logs from log directory.
        log_path = os.path.join(self.log_dir, self.name)
        if self.process_log_dir and not os.path.exists(log_path):
            try:
                os.symlink(self.process_log_dir, log_path)
            except OSError as e:
                Output.log(
                    "Failed to create symlink from {} to {}".format(self.process_log_dir, log_path),
                    logging.ERROR)

    # Records given pid in pidfile.
    # TODO: Redirect YW logs to yugabyte-logs
    def write_pid(self, pid):
        with open(self.pidfile, "w+") as pid_file:
            pid_file.write(str(pid))
            Output.log("{} started running with PID {}.".format(self.name, pid))

    # Returns pid of this process if it's running.
    def get_pid(self):
        if os.path.exists(self.pidfile):
            if self.process:
                return self.process.pid
            else:
                with open(self.pidfile, "r") as f:
                    try:
                        pid = int(f.readline())
                    except ValueError as e:
                        Output.log(
                            "Could not parse int PID from {}. Deleting file.".format(self.pidfile),
                            logging.DEBUG)
                        self.delete_pidfile()
                        return None
                command = ProcessManager.get_command(pid)
                if command and self.name.encode('utf8') in command:
                    return pid

            Output.log(
                "Pidfile {} was not properly deleted."
                "Contained PID {}. Deleting file.".format(self.pidfile, pid), logging.DEBUG)
            self.delete_pidfile()
        return None

    # Kills self.process if it exists.
    def kill(self):
        err = None
        pid = None
        if self.process:
            self.process.kill()
        else:
            pid = self.get_pid()
            if pid:
                try:
                    os.kill(pid, SIGTERM)
                except OSError as e:
                    return (e, pid)
        self.delete_pidfile()
        return (err, pid)

    # Raise RetryableError if pidfile still exists.
    def is_proc_running(self, pid):
        if (os.path.exists(self.pidfile) or ProcessManager.get_command(pid) != ""):
            raise RetryableError()
        else:
            return True

    # Function that waits until pidfile no longer exists.
    def wait_until_stop(self, pid):
        def temp_func():
            return self.is_proc_running(pid)
        retry_op(temp_func, timeout=60)
        return

    # Delete corresponding pidfile for this process.
    def delete_pidfile(self):
        if os.path.exists(self.pidfile):
            try:
                os.remove(self.pidfile)
            except OSError as e:
                if os.path.exists(self.pidfile):
                    Output.log(
                        "Failed to delete {}.".format(self.pidfile), level=logging.ERROR)
        self.start_time = None

    # Check fatal errors in fatal/error logs, if any. Overwritten in YBProcessManager
    def check_fatals(self):
        pass

    # Returns process status.
    def is_running(self):
        self.check_fatals()
        return self.get_pid() and self.process and self.process.poll() is None

    # Checks resource settings for current shell. Prints warning if requirements aren't met.
    def set_rlimits(self, print_info=False):
        rlim_max = resource.RLIM_INFINITY
        # TODO: Figure out what specs are recommended. max_user_processes is problematic.
        # https://github.com/yugabyte/yugabyte-db/issues/2818
        recommended_resources = {
            # "RLIMIT_FSIZE": (rlim_max, "file_size"),
            # "RLIMIT_MEMLOCK": (rlim_max, "max_locked_memory"),
            # "RLIMIT_AS": (rlim_max, "max_memory_size"),
            "RLIMIT_NOFILE": (1048576, "open_files"),
            # "RLIMIT_CPU": (rlim_max, "cpu_time"),
            "RLIMIT_NPROC": (MAX_PROC[OS_NAME], "max_user_processes"),
            # "RLIMIT_VMEM": (rlim_max, "virtual_memory"),
        }

        # If the current platform does not support the resource,
        # it won't be defined in the resource module.
        failed = []
        for res, (min_val, ulimit) in recommended_resources.items():
            if not hasattr(resource, res):
                continue
            # Check soft limit, not hard limit.
            key = getattr(resource, res)
            soft_lim, hard_lim = resource.getrlimit(key)
            if soft_lim != rlim_max and (soft_lim < min_val or min_val == rlim_max):
                try:
                    resource.setrlimit(key, (min_val, hard_lim))
                    if print_info:
                        Output.log("Changed {} from {} to {}".format(res, soft_lim, min_val))
                except ValueError as e:
                    failed.append((ulimit, soft_lim, min_val))
                    if print_info:
                        Output.log(
                            "Error changing {} from {} to {}: {}".format(
                                res, soft_lim, min_val, e),
                            logging.ERROR)

        if failed and print_info:
            return list(zip(*failed))[0]

    # Returns the command that was run with the input pid.
    @staticmethod
    def get_command(pid):
        try:
            return subprocess.check_output(["ps", "-p", str(pid), "-o", "command="])
        except subprocess.CalledProcessError as e:
            return ""

    # Returns if process called name is running.
    @staticmethod
    def is_process_running(name, pid_dir):
        return ProcessManager(name, cmd="", log_dir="", data_dir=pid_dir).get_pid() is not None


# Class for managing yugabyted process components - e.g. pidfiles and lockfiles.
# Maybe this class can take over the ControlScript.start_processes functionality?
class ScriptProcessManager(ProcessManager):
    def __init__(self, log_dir, data_dir):
        super(ScriptProcessManager, self).__init__(SCRIPT_NAME, "", log_dir, data_dir)
        # Used to retrieve status of daemon process. When daemon successfully initializes,
        # it will put a value here which can be checked on.
        self.daemon_success = multiprocessing.Queue()

    def start(self):
        return

    def is_running(self):
        pid = self.get_pid()
        # In certain scenarios like docker, the pid of a previously crashed docker run
        # is going to be 1, which is the same as ours and looks like the previous run is
        # still ongoing. The getpid check below covers that case.
        # In the long run, the plan is to move to something like flock instead.
        return pid is not None and pid != os.getpid()


class YBProcessManager(ProcessManager):
    def __init__(self, name, cmd, log_dir, data_dir):
        data_log_path = "{}/yb-data/{}/logs".format(data_dir, name)
        super(YBProcessManager, self).__init__(name, cmd, log_dir, data_dir, data_log_path)
        self.error_log = "{}/yb-{}.ERROR".format(data_log_path, name)


    def start(self):
        # Remove old logs as timestamped logs should have already been created.
        self.remove_error_logs()

        super(YBProcessManager, self).start()

    def remove_error_logs(self):
        if os.path.isfile(self.error_log):
            os.remove(self.error_log)

    def check_fatals(self):
        # Error logs contain port information, but fatal logs don't.
        address_error_1 = "Could not start on address"
        address_error_2 = "Error binding socket to "
        address_error_3 = "Is another postmaster already running on port "
        if os.path.isfile(self.error_log):
            with open(self.error_log) as log:
                for line in log.readlines():
                    if address_error_1 in line:
                        err_msg = line.split(address_error_1)
                        # Try to find address, otherwise log entire error message.
                        if len(err_msg) == 2:
                            err_msg = err_msg[1]
                        else:
                            err_msg = line
                        Output.log_error_and_exit(
                            "Failed to bind to address: {}".format(err_msg))
                    elif address_error_2 in line:
                        err_msg = line.split(address_error_2)[1]
                        address = err_msg.split()[0]
                        Output.log_error_and_exit(
                            "Failed to bind to address: {}".format(address))
                    elif address_error_3 in line:
                        err_msg = line.split(address_error_3)
                        # Try to find address, otherwise log entire error message.
                        if len(err_msg) == 2:
                            err_msg = err_msg[1].split()[0]
                            Output.log_error_and_exit(
                                "Failed to bind to port: {}.".format(err_msg))
                        else:
                            Output.log_error_and_exit(
                                "Failed to bind to address: {}".format(err_msg))

@dataclass
class CallhomePayload:
    callhome_type: str
    command: str
    timestamp: str
    node_uuid: str
    cluster_uuid: str
    universe_uuid: str
    is_ui_up: bool
    is_read_replica: bool
    payload: dict

    def to_json(self):
        return json.dumps(self.__dict__)


class Diagnostics(object):
    first_install = None
    first_run_secs = None

    def __init__(self, configs):
        self.configs = configs

    # Collects callhome data and sends it.
    def callhome(self, callhome_payload: CallhomePayload):
        if self.configs.saved_data.get("callhome"):
            try:
                event_name = getattr(callhome_payload, "command", "callhome")
                Output.log("Preparing callhome event '{}' for diagnostics".format(event_name))
                url = "https://diagnostics.yugabyte.com"
                headers = {
                    "Content-Type": "application/json",
                    "User-Agent": "Mozilla",
                }
                data = callhome_payload.to_json()
                req = Request(url, headers=headers, data=data.encode('utf8'))
                _ = urlopen(req)
                Output.log("Callhome event '{}' sent.".format(event_name))
            except Exception as e:
                Output.log("Callhome failed: " + str(e))
                pass


class YBTableServerCLIProxy(object):
    cmd_args = []

    @staticmethod
    def init():
        YBTableServerCLIProxy.cmd_args.append(find_binary_location("yb-ts-cli"))

    @staticmethod
    def set_certs_dir(master_flags, secure, certs_dir):
        # If the user is attempting to use TLS, let's point yb-admin to
        # the same certs dir as the master
        if secure:
            YBTableServerCLIProxy.cmd_args.append("--certs_dir_name={}".format(certs_dir))
        elif master_flags:
            flags_list = master_flags.split(",")
            if 'use_node_to_node_encryption=true' not in flags_list:
                return
            certs_dir_name = [y for y in
                [re.match('certs_dir=(.*)', x) for x in flags_list]
                if y is not None]
            if not certs_dir_name[0]:
                raise RuntimeError("use_node_to_node_encryption=true must "
                                "be accompanied by a certs_dir setting")
            YBTableServerCLIProxy.cmd_args.append('--certs_dir_name={}'.
                                                 format(certs_dir_name[0].group(1)))

    @staticmethod
    def update_gflag_in_tserver(tserver_ip, tserver_port, gflag, gflag_value, byForce=False):
        cmd = YBTableServerCLIProxy.cmd_args + \
            [
                "--server_address={}:{}".format(get_url_from_ip(tserver_ip),
                                                tserver_port),
                "set_flag",
                gflag,
                "{}".format(gflag_value)
            ]

        if byForce:
            cmd = cmd + \
                [
                    "--force"
                ]

        out, err, ret_code = run_process(cmd, timeout=10, log_cmd=True)
        return (0 == ret_code)

class YBControllerCLIProxy(object):
    cmd_args = []

    @staticmethod
    def init():
        YBControllerCLIProxy.cmd_args.append(find_ybc_binary_location("yb-controller-cli"))

    @staticmethod
    def set_certs_dir(master_flags, secure, certs_dir):
        # If the user is attempting to use TLS, let's point yb-admin to
        # the same certs dir as the master
        if secure:
            YBControllerCLIProxy.cmd_args.append("--certs_dir_name={}".format(certs_dir))
        elif master_flags:
            flags_list = master_flags.split(",")
            if 'use_node_to_node_encryption=true' not in flags_list:
                return
            certs_dir_name = [y for y in
                [re.match('certs_dir=(.*)', x) for x in flags_list]
                if y is not None]
            if not certs_dir_name[0]:
                raise RuntimeError("use_node_to_node_encryption=true must "
                                "be accompanied by a certs_dir setting")
            YBControllerCLIProxy.cmd_args.append('--certs_dir_name={}'.
                                                 format(certs_dir_name[0].group(1)))

    @staticmethod
    def backup_to_cloud(tserver_ip, ybc_cli_flags):

        cmd = YBControllerCLIProxy.cmd_args + \
        [
                "backup",
                "--cloud_type={}".format(ybc_cli_flags["cloud_type"]),
                "--bucket={}".format(ybc_cli_flags["bucket"]),
                "--cloud_dir={}".format(ybc_cli_flags["cloud_dir"]),
                "--tserver_ip={}".format(tserver_ip),
                "--ns={}".format(ybc_cli_flags["ns"]),
                "--ns_type={}".format(ybc_cli_flags["ns_type"]),
        ]

        out, err, ret_code = run_process(cmd, timeout=10, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return []

        match = re.search("task:", out)
        if match:
            task_id = out[match.end():]
            task_id = task_id.lstrip()
            task_id = task_id.rstrip()
            return task_id

        return []

    @staticmethod
    def restore_from_cloud(tserver_ip, ybc_cli_flags):
        # ./bin/yb-controller-cli restore
        # --bucket nikhil.customer.share
        # --cloud_type nfs --tserver_ip 10.151.0.29
        # --ns yb_demo_northwind --ns_type ysql --wait
        # --cloud_dir ybc_test_08_04_2023_1

        cmd = YBControllerCLIProxy.cmd_args + \
        [
                "restore",
                "--cloud_type={}".format(ybc_cli_flags["cloud_type"]),
                "--bucket={}".format(ybc_cli_flags["bucket"]),
                "--cloud_dir={}".format(ybc_cli_flags["cloud_dir"]),
                "--tserver_ip={}".format(tserver_ip),
                "--ns={}".format(ybc_cli_flags["ns"]),
                "--ns_type={}".format(ybc_cli_flags["ns_type"]),
        ]

        out, err, ret_code = run_process(cmd, timeout=10, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return []

        match = re.search("task:", out)
        if match:
            task_id = out[match.end():]
            task_id = task_id.lstrip()
            task_id = task_id.rstrip()
            return task_id

        return []

    # Records the backup task id in backup.lock
    @staticmethod
    def write_ybc_lock_file(ybc_args, task_id, ybc_dir, tserver_ip):
        # "{}/yb-data/ybc".format(data_dir)
        backup_lock = os.path.join(
            ybc_dir,
            "{}_{}_{}.lock".format(ybc_args.get("ns"), ybc_args.get("ns_type"),
                                   ybc_args.get("ybc_task")))
        with open(backup_lock, "w+") as backup_lock_file:
            backup_lock_file.write(
                str(task_id + "," + tserver_ip + "," + ybc_args.get("ybc_task") + ","
                    + ybc_args.get("ns_type") + "," + ybc_args.get("ns") + ","
                    + datetime.utcnow().strftime('%B %d %Y - %H:%M:%S')))
            Output.log("Started backup with task id {}.".format(task_id))

    @staticmethod
    def rename_ybc_lock_file(ybc_args, ybc_dir):
        # "{}/yb-data/ybc".format(data_dir)
        backup_lock = os.path.join(
            ybc_dir, "{}_{}_{}.lock".format(ybc_args.get("ns"), ybc_args.get("ns_type"),
                                            ybc_args.get("ybc_task"))
        )
        rename_filename = os.path.join(
            ybc_dir, "{}_{}_{}.{}".format(ybc_args.get("ns"), ybc_args.get("ns_type"),
                                          ybc_args.get("ybc_task"), time.time())
        )
        try:
            os.rename(backup_lock, rename_filename)
        except OSError as e:
            Output.log("Unable to delete the backup lock file. " + str(e))

    @staticmethod
    def check_for_ongoing_ybc_task(ybc_args, ybc_dir):
        ybc_lock = os.path.join(
            ybc_dir,
            "{}_{}_{}.lock".format(ybc_args.get("ns"), ybc_args.get("ns_type"),
                                   ybc_args.get("ybc_task")))
        ybc_task_details = []
        if os.path.isfile(ybc_lock):
            with open(ybc_lock, 'r') as file:
                line = file.readline()
                ybc_task_details = line.split(',')
                Output.log("{}: checking for ongoing ybc task {},{}".
                           format(ybc_args.get("ybc_task"),
                                  ybc_task_details[0],
                                  ybc_task_details[1])
                           )
        else:
             return None

        backup_status = YBControllerCLIProxy.get_ybc_task_status(
            ybc_task_details[0], ybc_task_details[1])

        return backup_status

    @staticmethod
    def check_for_existing_backup(ybc_cli_flags):

        # ./bin/yb-controller-cli download_md
        # --cloud_dir ybc_test_08_04_2023_1
        # --bucket nikhil.customer.share
        # --cloud_type nfs --local_dir /tmp/ybc_test_08_04_2023_1
        cmd = YBControllerCLIProxy.cmd_args + \
            [
                "download_md",
                "--cloud_dir={}".format(ybc_cli_flags["cloud_dir"]),
                "--bucket={}".format(ybc_cli_flags["bucket"]),
                "--cloud_type={}".format(ybc_cli_flags["cloud_type"]),
                "--local_dir=/tmp/{}".format(ybc_cli_flags["cloud_dir"])
            ]
        out, err, ret_code = run_process(cmd, timeout=10, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return False

        match = re.search("Status:", out)
        if match:
            download_md_status = out[match.end():]
            download_md_status = download_md_status.lstrip()
            download_md_status = download_md_status.rstrip()
            if download_md_status == "OK":
                Output.log("Existing backup found at - {}".format(
                    ybc_cli_flags["cloud_dir"]))
                return True

        return False

    @staticmethod
    def get_ybc_task_status(taskid, tserver_ip):

        cmd = YBControllerCLIProxy.cmd_args + \
            [
                "task_progress",
                "--task_id={}".format(taskid),
                "--tserver_ip={}".format(tserver_ip),
            ]
        out, err, ret_code = run_process(cmd, timeout=10, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return []

        match = re.search("Final Status:", out)
        if match:
            ybc_task_status = out[match.end():]
            ybc_task_status = ybc_task_status.lstrip()
            ybc_task_status = ybc_task_status.rstrip()
            Output.log("backup: task status {} - {}".format(taskid, ybc_task_status))
            return ybc_task_status

        return []

class PGUpgradeProxy(object):
    cmd_args = []

    @staticmethod
    def init():
        PGUpgradeProxy.cmd_args.append(find_binary_location("pg_upgrade"))

    @staticmethod
    def check_for_pg_upgrade_compatibility(yugabyted_advertise_address,
                    ysql_rpc_port, data_dir, timeout=3600):
        upgrade_error_message = ""
        upgrade_return_code = 1
        cmd = PGUpgradeProxy.cmd_args + \
            [
                "--check",
                "-U{}".format('yugabyte'),
                "--old-port={}".format(ysql_rpc_port),
                "--old-host={}".format(yugabyted_advertise_address),
                "--old-datadir={}".format(data_dir + "/pg_data_11"),
            ]
        out, err, ret_code = run_process(cmd, timeout, log_cmd=True)

        if 0 != ret_code or len(out.splitlines()) <=1:
            upgrade_error_message = out
            upgrade_return_code = 1
            return [upgrade_error_message, upgrade_return_code]

        match = re.search(r"Clusters are compatible", out)
        if match:
            upgrade_return_code = 0
            Output.log("Cluster is compatible for pg15 upgrade.")
            return [upgrade_error_message, upgrade_return_code]
        else:
            upgrade_error_message = "Cluster is not compatible for pg15 upgrade."
            Output.log(upgrade_error_message)
            return [upgrade_error_message, upgrade_return_code]

# Proxy for parsing output from yb-admin commands.
class YBAdminProxy(object):
    cmd_args = []

    @staticmethod
    def init():
        YBAdminProxy.cmd_args.append(find_binary_location("yb-admin"))

    @staticmethod
    def set_certs_dir(master_flags, secure, certs_dir, hostname):
        # If the user is attempting to use TLS, let's point yb-admin to
        # the same certs dir as the master

        if secure:
            YBAdminProxy.cmd_args.append("--certs_dir_name={}".format(certs_dir))

        if master_flags:

            flags_list = master_flags.split(",")

            if not secure:
                if 'use_node_to_node_encryption=true' not in flags_list:
                    return
                certs_dir_name = [y for y in
                    [re.match('certs_dir=(.*)', x) for x in flags_list]
                    if y is not None]
                if not certs_dir_name[0]:
                    raise RuntimeError("use_node_to_node_encryption=true must "
                                    "be accompanied by a certs_dir setting")
                YBAdminProxy.cmd_args.append(
                  '--certs_dir_name={}'.format(certs_dir_name[0].group(1)))

            if 'node_to_node_encryption_use_client_certificates=true' in flags_list:
                YBAdminProxy.cmd_args.append('--client_node_name={}'.format(hostname))

    @staticmethod
    def add_master(master_addrs, new_master_ip, new_master_rpc_port, timeout=30):
        cmd = YBAdminProxy.cmd_args + ["--init_master_addrs", master_addrs,
              "change_master_config", "ADD_SERVER", new_master_ip, str(new_master_rpc_port)]
        out, err, ret_code = run_process_with_retries(cmd=cmd, timeout=timeout, log_cmd=True)
        return (0 == ret_code)

    @staticmethod
    def remove_master(master_addrs, old_master, old_master_rpc_port, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--init_master_addrs", master_addrs,
              "change_master_config", "REMOVE_SERVER", old_master, str(old_master_rpc_port)]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (0 == ret_code)

    @staticmethod
    def set_rf(master_addrs, rf, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--init_master_addrs", master_addrs,
                "modify_placement_info", "cloud1.datacenter1.rack1", str(rf) ]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (0 == ret_code)

    @staticmethod
    # Returns [ (uuid, ip:port, role) ] for each master
    def get_masters(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--master_addresses", master_addrs, "list_all_masters"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return []
        masters = [ line.split() for line in out.splitlines()[1:] ]
        return [ (master[0], master[1], master[3]) for master in masters ]

    # Returns list[tserver uuid] reported by yb-master.
    @staticmethod
    def get_tservers(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--master_addresses", master_addrs,
                "list_all_tablet_servers"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if 0 != ret_code or len(out.splitlines()) <= 1:
            return None
        return [ line.split()[0] for line in out.splitlines()[1:] ]

    # Returns the cluster_uuid for this universe
    @staticmethod
    def get_cluster_uuid(master_addrs):
        cluster_config = YBAdminProxy.get_cluster_config(master_addrs)
        if cluster_config:
            return cluster_config['clusterUuid']
        else:
            return None

    # Returns the rf for this universe
    @staticmethod
    def get_cluster_rf(master_addrs):
        cluster_config = YBAdminProxy.get_cluster_config(master_addrs)
        if cluster_config:
            if "replicationInfo" in cluster_config:
                return cluster_config["replicationInfo"].get("liveReplicas").get("numReplicas")
            else:
                return 1
        else:
            return None

    # Returns the cluster config for this universe
    @staticmethod
    def get_cluster_config(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--master_addresses", master_addrs, "get_universe_config"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if ret_code:
            return None
        try:
            return json.loads(out)
        except Exception:
            return None

    # Returns node_uuid by finding the UUID corresponding to current master's IP
    @staticmethod
    def get_node_uuid(master_addrs, cur_master_addr):
        cmd = YBAdminProxy.cmd_args + ["--init_master_addrs", master_addrs, "list_all_masters"]
        out, err, ret_code = run_process(cmd)
        if ret_code:
            return None
        for line in out.splitlines()[1:]:
            master_uuid, rpc_addr, _, _ = line.split()
            if rpc_addr == cur_master_addr:
                return master_uuid
        return None

    # Sets placement info
    @staticmethod
    def modify_placement_info(master_addrs, placement_locations, placement_uuid,
                              replication_factor = "3", timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs, "modify_placement_info",
                placement_locations, replication_factor, placement_uuid]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    @staticmethod
    def set_preferred_zones(master_addrs, priority_info, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses",
                                       master_addrs, "set_preferred_zones"] + priority_info
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return ret_code == 0

    # Sets placement info of read replicas
    @staticmethod
    def set_read_replica_placement(master_addrs, placement_locations, placement_uuid,
                              replication_factor, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                        "add_read_replica_placement_info",placement_locations,
                        replication_factor, placement_uuid]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Modify placement info of read replicas
    @staticmethod
    def modify_read_replica_placement(master_addrs, placement_locations, placement_uuid,
                              replication_factor, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                        "modify_read_replica_placement_info",placement_locations,
                        replication_factor, placement_uuid]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Delete placement info of read replicas
    @staticmethod
    def delete_read_replica_placement(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                        "delete_read_replica_placement_info"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Copy universe key to all masters for encryption at rest
    @staticmethod
    def copy_key_to_masters(master_addrs, key_id, key_path, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
            "add_universe_key_to_all_masters", key_id, key_path]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Check if universe key has been copied to all masters for encryption at rest
    @staticmethod
    def check_key_in_masters(master_addrs, key_id, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
            "all_masters_have_universe_key_in_memory", key_id]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if 0 != ret_code:
            return []
        out = out.splitlines()
        return out

    # Enable encryption at rest by start using the key in masters
    @staticmethod
    def enable_encryption_using_key(master_addrs, key_id, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
            "rotate_universe_key_in_memory", key_id]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Disable encryption at rest
    @staticmethod
    def disable_encryption(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs, "disable_encryption"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Check if universe key has been copied to all masters for encryption at rest
    @staticmethod
    def check_encryption(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs, "is_encryption_enabled"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if 0 != ret_code:
            return None
        return out

    # Enable point_in_time_recovery
    @staticmethod
    def create_snapshot_schedule(master_addrs, retention,
                pitr_object, timeout=10):
        cmd = YBAdminProxy.cmd_args + [
            "--master_addresses", master_addrs, "create_snapshot_schedule",
            "1440", str(retention*1440), pitr_object]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return ret_code == 0

    # Disable point_in_time_recovery
    @staticmethod
    def delete_snapshot_schedule(master_addrs, pitr_object_id, timeout=10):
        cmd = YBAdminProxy.cmd_args + [
            "--master_addresses", master_addrs, "delete_snapshot_schedule", pitr_object_id]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return ret_code == 0

    # List point_in_time recovery configuration details
    @staticmethod
    def list_schedules(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["--master_addresses", master_addrs,
                                        "list_snapshot_schedules"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if ret_code:
            return None
        try:
            data = json.loads(out)
            # Return an empty list if there are no schedules,
            # rather than None, to signify no schedules found without error
            return data.get("schedules", [])
        except Exception:
            return None

    # Restore to a specific point in time
    @staticmethod
    def restore_snapshot_schedule(master_addrs, pitr_object_id, restore_time, timeout=10):
        cmd = YBAdminProxy.cmd_args + [
            "--master_addresses", master_addrs, "restore_snapshot_schedule", pitr_object_id,
                                            restore_time]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return ret_code, err

    # Promote Auto Flags
    @staticmethod
    def promote_auto_flags(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs, "promote_auto_flags"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (ret_code == 0)

    # Upgrade YSQL catalog
    @staticmethod
    def upgrade_ysql(master_addrs, timeout):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
        "-timeout_ms", str(timeout), "upgrade_ysql"]
        out, err, ret_code = run_process(cmd, log_cmd=True)
        return (ret_code == 0)

    # Finalize the YSQL major catalog upgrade
    @staticmethod
    def finalize_upgrade_uber_command(master_addrs, timeout=3600):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
        "-timeout_ms", str(timeout), "finalize_upgrade"]
        out, err, ret_code = run_process(cmd, log_cmd=True, timeout=timeout)
        return (ret_code == 0)

    @staticmethod
    def create_checkpoint_xcluster(master_addrs, replication_id, databases,
                                   automatic_mode, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "create_xcluster_checkpoint", replication_id, databases]
        if automatic_mode:
            cmd += ["automatic_ddl_mode"]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, err, ret_code)

    @staticmethod
    def add_to_checkpoint_xcluster(master_addrs, replication_id, database, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "add_namespace_to_xcluster_checkpoint",
                                       replication_id, database]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, err, ret_code)

    @staticmethod
    def get_source_xcluster_replication_ids(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "list_xcluster_outbound_replication_groups"]
        out, _, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if ret_code:
            return None

        matches = re.findall(r'\[(.*?)\]', out)
        replication_ids = [replication_id.strip() for replication_id in matches[0].split(',')]

        return replication_ids

    @staticmethod
    def get_source_xcluster_databases(master_addrs, replication_id, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "get_xcluster_outbound_replication_group_info",
                                       replication_id]
        out, _, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if ret_code:
            return None

        databases_with_ids = re.findall(r'Namespace ID:\s*(\w+)\s*Namespace name:\s*(\w+)', out)
        # databases_data is a map with <db_name>:<db_id> as the format
        databases_data = dict()
        for database_id, database in databases_with_ids:
            databases_data[database] = database_id

        return (databases_data)

    @staticmethod
    def xcluster_is_bootstrap_required(master_addrs, replication_id, databases, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "is_xcluster_bootstrap_required", replication_id, databases]
        out, _, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, ret_code)

    @staticmethod
    def set_up_xcluster(master_addrs, replication_id, target_addresses, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "setup_xcluster_replication", replication_id,
                                       target_addresses]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, err, ret_code)

    @staticmethod
    def add_to_replication_xcluster(master_addrs, replication_id, database, target_addresses,
                                    timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "add_namespace_to_xcluster_replication", replication_id,
                                       database, target_addresses]
        out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, err, ret_code)

    @staticmethod
    def get_xcluster_safe_time(master_addrs, timeout=10):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "get_xcluster_safe_time",
                                       "include_lag_and_skew"]
        out, _, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        if ret_code == 0:
            return json.loads(out)
        else:
            return None

    @staticmethod
    def drop_xcluster_replication(master_addrs, replication_id, target_addresses = '', timeout=10):
        if target_addresses == '':
            cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "drop_xcluster_replication", replication_id]
        else:
            cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "drop_xcluster_replication", replication_id,
                                       target_addresses]

        _, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (err, ret_code)

    @staticmethod
    def delete_from_xcluster_replication(master_addrs, replication_id, database,
                                        target_addresses = '', timeout=10):
        if target_addresses == '':
            cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "remove_namespace_from_xcluster_replication",
                                       replication_id, database]
        else:
            cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "remove_namespace_from_xcluster_replication",
                                       replication_id, database, target_addresses]

        _, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (err, ret_code)

    @staticmethod
    def start_ysql_catalog_upgrade(master_addrs, timeout=300000):
        cmd = YBAdminProxy.cmd_args + ["-master_addresses", master_addrs,
                                       "upgrade_ysql_major_version_catalog"]
        out, _, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
        return (out, ret_code)


    @staticmethod
    def check_for_ysql_catalog_upgrade_state(master_addrs, timeout=60):

        upgrade_error_message = ""
        upgrade_return_code = 1
        cmd = YBAdminProxy.cmd_args + \
            [
                "-master_addresses", master_addrs,
                "get_ysql_major_version_catalog_state"
            ]
        out, err, ret_code = run_process(cmd, timeout, log_cmd=True)

        if 0 != ret_code or len(out.splitlines()) < 1:
            upgrade_error_message = out
            upgrade_return_code = 1
            return [upgrade_error_message, upgrade_return_code]

        match = re.search(r"awaiting finalization", out)
        if match:
            upgrade_return_code = 0
            return [upgrade_error_message, upgrade_return_code]

        return [upgrade_error_message, upgrade_return_code]


    # Passthrough method for all the yb-admin commands
    # @staticmethod
    # def call_yb_admin_command(master_addrss, command, timeout=10):
    #     cmd =  YBAdminProxy.cmd_args + ["-master_addresses", master_addrss, command]
    #     out, err, ret_code = run_process(cmd, timeout=timeout, log_cmd=True)
    #     if 0 != ret_code:
    #         return None
    #     return out


# Proxy for ysqlsh commands.
class YsqlProxy(object):
    def __init__(self, ip, port, path=None,
                    get_default_credentials=False):
        if path is None:
            path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ysql"])
        self.setup_env_init = EnvBasedCredentials()
        self.username, self.password, self.db = self.setup_env_init.get_ysql_credentials(
                                                    get_default_credentials)
        self.cmd_with_password = [path, "postgresql://{}:{}@{}:{}".format(self.username,
                self.password, get_url_from_ip(ip), port)]
        self.cmd_without_password = [path, "-h", str(ip), "-p", str(port)]
        env_var = os.environ.copy()
        env_var["PGUSER"] = self.username
        env_var["PGPASSWORD"] = self.password
        env_var["PGDATABASE"] = self.db

        self.env = env_var

    # Starts interactive YSQL shell.
    def connect(self, cmd):
        env_var = os.environ.copy()
        env_var["PGUSER"] = self.username
        Output.log("Connecting to the YSQL database using command: {}" \
                             .format(cmd))
        shell = subprocess.Popen(cmd, env=env_var)
        while True:
            try:
                shell.communicate()
            except KeyboardInterrupt:
                continue
            break

    def connect_without_password(self, db=None):
        if db is None:
            db = self.db

        cmd = self.cmd_without_password + ["-d", db]
        self.connect(cmd)

    def connect_with_password(self, db=None):
        if db is None:
            db = self.db

        cmd=list(self.cmd_with_password)
        cmd[1] +="/"+db
        self.connect(cmd)

    # Checks if db exists.
    # Note that this will return false if ysqlsh can't connect, even if db exists.
    def db_exists(self, db):
        cmd = self.cmd_with_password + ["-q", "-c", "\\t", "-c",
            "select datname from pg_catalog.pg_database where datname='{}'".format(db)]
        return db in run_process_checked(cmd=cmd, env_vars=self.env).strip()

    # Creates specified db.
    def create_db(self, db):
        cmd = self.cmd_with_password + ["-c", "create database \"{}\"".format(db)]
        run_process_checked(cmd=cmd, env_vars=self.env)

    # Deletes specified db.
    def drop_db(self, db):
        cmd = self.cmd_with_password + ["-c", "drop database {}".format(db)]
        run_process_checked(cmd=cmd, env_vars=self.env)

    # Runs ysqlsh with specified files.
    def load_files(self, filepaths, db=None):
        cmd = list(self.cmd_with_password)
        env = self.env
        if db:
            env['PGDATABASE'] = db
        else:
            env['PGDATABASE'] = self.db
        for path in filepaths:
            cmd.extend(["-f", path])
        run_process_checked(cmd=cmd, log_cmd=False, env_vars=env)

    # Check user exists
    # Note that this will return false if ysqlsh can't connect, even if user exists.
    def user_exists(self, ysql_username):
        cmd = self.cmd_with_password + \
                ["-c", "select rolname from pg_catalog.pg_roles where rolname='{}'".
                format(ysql_username)]
        run_process_checked(cmd=cmd, env_vars=self.env).strip() == ysql_username

    # Create specified superuser
    def create_user(self, ysql_username, ysql_password):
        cmd = self.cmd_with_password + \
                ["-c", "create role \"{}\" with LOGIN SUPERUSER password '{}';".
                format(ysql_username, ysql_password)]
        run_process_checked(cmd=cmd, log_cmd=False, env_vars=self.env)

    # Delete specified user
    def delete_user(self, username, password, user_to_delete=DEFAULT_YSQL_USER):
        cmd = self.cmd_with_password + ["-c", "drop role {};".format(user_to_delete)]
        env = self.env
        env['PGUSER'] = username
        env['PGPASSWORD'] = password
        run_process_checked(cmd=cmd, env_vars=env)

    # Update specified user's password
    def update_password(self, new_password, user_to_update=DEFAULT_YSQL_USER):
        cmd = self.cmd_with_password + \
            ["-c", "alter role {} password '{}';".format(user_to_update, new_password)]
        run_process_checked(cmd=cmd, log_cmd=False, env_vars=self.env)

    # Try to update specified user's password otherwise throw Retryable Error
    def try_update_password(self, new_password, user_to_update=DEFAULT_YSQL_USER):
        cmd = self.cmd_with_password + \
            ["-c", "alter role {} password '{}';".format(user_to_update, new_password)]

        encrypted_password = "*" * len(new_password)
        encrypted_cmd = self.cmd_with_password + \
            ["-c", "alter role {} password '{}';".format(user_to_update, encrypted_password)]

        out, err, retcode = run_process(cmd=cmd, encrypted_cmd=encrypted_cmd, log_cmd=True,
                env_vars=self.env)
        if (retcode != 0):
            raise RetryableError

    # Change specified DB's owner
    def db_owner(self, db, new_owner):
        cmd = self.cmd_with_password + \
            ["-c", "alter database \"{}\" owner to \"{}\";".format(db, new_owner)]
        run_process_checked(cmd=cmd, env_vars=self.env)

    # Check YSQL is UP
    def is_ysql_up(self):
        cmd = self.cmd_with_password + ["-c", "\\conninfo"]
        out, err, _ = run_process(cmd=cmd, log_cmd=True, env_vars=self.env)
        if err:
            Output.log("Unable to connect using YSQL. Error: {}".format(err))
            raise RetryableError()
        else:
            Output.log("YSQL Connection Info - {}".format(out))
            return True

# Proxy for ycqlsh commands.
class YcqlProxy(object):
    def __init__(self, ip, port, path=None,
                    get_default_credentials=False, secure=False):
        if path is None:
            path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ycql"])
        self.cmd = [path, str(ip), str(port)]
        self.setup_env_init = EnvBasedCredentials()
        self.username, self.password, self.keyspace = self.setup_env_init.get_ycql_credentials(
                                                        get_default_credentials)
        self.password_authentication=secure

        if secure:
            self.cmd.append("--ssl")

    # Starts interactive YCQL shell.
    def connect(self, cmd):
        Output.log("Connecting to the YCQL keyspace using command: {}" \
                             .format(cmd))
        shell = subprocess.Popen(cmd)
        while True:
            try:
                shell.communicate()
            except KeyboardInterrupt:
                continue
            break

    def connect_without_password(self):
        cmd = list(self.cmd)
        if self.password_authentication:
            cmd.extend(["-u", self.username])
        if self.keyspace is not None:
            cmd.extend(["-k", self.keyspace])
        self.connect(cmd)

    def connect_with_password(self):
        cmd = list(self.cmd)
        if self.username:
            cmd.extend(["-u", self.username])
        if self.keyspace is not None:
            cmd.extend(["-k", self.keyspace])
        cmd.extend(["-p", self.password])
        self.connect(cmd)

    # Check user exists
    # Note that this will return false if ycqlsh can't connect, even if user exists.
    def user_exists(self, ycql_username):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
            "SELECT role FROM system_auth.roles WHERE role='{}';".format(ycql_username)]
        return run_process_checked(cmd).strip() == ycql_username

    # Create specified superuser
    def create_user(self, ycql_username, ycql_password):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
                "CREATE ROLE \"{}\" WITH PASSWORD = '{}' AND LOGIN = true AND SUPERUSER = true;".
                format(ycql_username, ycql_password)]
        run_process_checked(cmd, log_cmd=False)

    # Delete specified user
    def delete_user(self, username, password, user_to_delete=DEFAULT_YCQL_USER):
        cmd = self.cmd + ["-u", username, "-p", password, "-e",
                "DROP ROLE IF EXISTS {};".format(user_to_delete)]
        run_process_checked(cmd)

    # Update specified user's password
    def update_password(self, new_password, user_to_update=DEFAULT_YCQL_USER):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
                "ALTER ROLE {} WITH PASSWORD = '{}';".format(user_to_update, new_password)]
        run_process_checked(cmd, log_cmd=False)

    # Try to update specified user's password otherwise throw Retryable Error
    def try_update_password(self, new_password, user_to_update=DEFAULT_YCQL_USER):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
                "ALTER ROLE {} WITH PASSWORD = '{}';".format(user_to_update, new_password)]

        encrypted_password = "*" * len(new_password)
        encrypted_cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
                "ALTER ROLE {} WITH PASSWORD = '{}';".format(user_to_update, encrypted_password)]

        out, err, retcode = run_process(cmd=cmd, encrypted_cmd=encrypted_cmd, log_cmd=True,
                env_vars=os.environ.copy())
        if (retcode != 0):
            raise RetryableError

    # Check keyspace exists
    # Note that this will return false if ycqlsh can't connect, even if keyspace exists.
    def keyspace_exists(self, keyspace):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
            "SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name='{}';".
            format(keyspace)]
        output = run_process_checked(cmd=cmd)
        if keyspace in output:
            return True

        return False

    # Create specified keyspace
    def create_keyspace(self, keyspace):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e",
                "CREATE KEYSPACE \"{}\";".format(keyspace)]
        run_process_checked(cmd)

    # Runs ycqlsh with specified files.
    # Example:
    # 1. bin/ycqlsh -f directory/a.ycql
    # 2. If environment variables exists: bin/ycqlsh -u user -p password -f directory/b.ycql
    def load_files(self, filepaths):
        cmd = self.cmd
        cmd.extend(["-u", self.username, "-p", self.password])
        for path in filepaths:
            cmd.extend(["-f", path])
        run_process_checked(cmd=cmd, log_cmd=False)

    # Check YCQL is UP
    def is_ycql_up(self):
        cmd = self.cmd + ["-u", self.username, "-p", self.password, "-e", "SHOW HOST"]
        out, err, _ = run_process(cmd, log_cmd=True)
        if err:
            Output.log("Unable to connect using YCQL. Error: {}".format(err))
            raise RetryableError()
        else:
            Output.log("YCQL Connection Info - {}".format(out))
            return True


# Proxy for creating ssl certificates and keys using openssl
class OpenSSLProxy(object):
    cmd_args = []

    @staticmethod
    def init(path=None):
        if path is None:
            path = find_binary_location("openssl_proxy.sh")

        OpenSSLProxy.cmd_args = [path]

    # Generate root ca certificates
    @staticmethod
    def generate_root_ca_certs(root_certs_dir, timeout=60):
        cmd = OpenSSLProxy.cmd_args + ['generate-ca', '-rcp', root_certs_dir]
        out, err, ret = run_process(cmd, timeout=timeout, log_cmd=True)

        return (0 == ret)

    # Generate node server certificates
    @staticmethod
    def generate_node_server_certs(root_certs_dir, server_cert_dir, hostname, hostname_type,
                                    timeout=60):
        cmd = OpenSSLProxy.cmd_args + ['generate-server-cert', '-rcp', root_certs_dir,
            '-scp', server_cert_dir, '-hn', hostname, '-ht', hostname_type]
        out, err, ret = run_process(cmd, timeout=timeout, log_cmd=True)

        return (0 == ret)

    # Generate universe key for encryption-at-rest
    @staticmethod
    def generate_key(key_dir, keyname, timeout=60):
        cmd = OpenSSLProxy.cmd_args + ['generate-key', '-kp', key_dir, '-kn', keyname]
        out, err, ret = run_process(cmd, timeout=timeout, log_cmd=True)

        return (0 == ret)


# Currently unused. Useful for getting diagnostics that are available only through logs.
class LogAnalyzer(object):
    unsupported_error = "not supported yet"
    def __init__(self, logfile):
        self.logfile = logfile
        # Flag to stop tailing the logfile.
        self.kill_thread = False
        self.unsupported_commands = []

    def analyze(self):
        lines = self.tail()
        for line in lines:
            if LogAnalyzer.unsupported_error in line:
                # Get the command logged right before error message
                cmd = line.split("not supported yet")[0].split()[-1]
                self.unsupported_commands.append(cmd)

    # Generator that continually returns last line of logfile.
    def tail(self):
        with open(self.logfile) as open_file:
            open_file.seek(0, 2)
            while not self.kill_thread:
                line = open_file.readline()
                if not line:
                    time.sleep(0.1)
                    continue
                yield line

# Manages API calls to YW.
class YugaWareProxy(object):
    def __init__(self, ip, webserver_port, univ_name="local-universe"):
        self.univ_name = univ_name
        self.api_token_secure = ""
        self.api_token_insecure = ""
        self.cust_uuid = ""
        self.url = "http://{}:{}/api/v1".format(get_url_from_ip(ip),
                                                webserver_port)

    # Retrieves permanent api_token. Returns error, if any.
    def login(self):
        try:
            target = "{}/login".format(self.url)
            headers = {
                "Content-Type": "application/json",
            }
            data = urlencode({"email": "admin", "password": "admin"})
            req = Request(target, data=data.encode('utf8'))
            resp = urlopen(req)
            session_data = json.loads(resp.read())
            auth_token = session_data["authToken"]
            self.cust_uuid = session_data["customerUUID"]
            # Auth token will expire, so get API token instead.
            target = "{}/customers/{}/api_token".format(self.url, self.cust_uuid)
            headers = {
                "X-Auth-Token": auth_token,
            }
            req = Request(target, headers=headers)
            req.get_method = lambda: "PUT"
            resp = urlopen(req)
            api_data = json.loads(resp.read())
            self.api_token_secure = api_data["apiToken"]
            return None
        except (ValueError, HTTPError, URLError, KeyError) as e:
            return "Failed to log into webserver. {}".format(e)

    # Attempts insecure login. Returns error, if any.
    def insecure_login(self):
        try:
            target = "{}/insecure_login".format(self.url)
            req = Request(target)
            resp = urlopen(req)
            session_data = json.loads(resp.read())
            self.api_token_insecure = session_data["apiToken"]
            self.cust_uuid = session_data["customerUUID"]
            return None
        except (ValueError, HTTPError, URLError, KeyError) as e:
            return "Failed to log into webserver. {}".format(e)

    # Import local universe into YW. Returns error, if any.
    def import_universe(self, master_address, master_rpc_port, universeUUID):
        target = "{}/customers/{}/universes/import".format(self.url, self.cust_uuid)
        headers = {
            "X-AUTH-YW-API-TOKEN": self.api_token_secure,
            "Content-Type": "application/json",
        }
        data = {
            "cloudProviderType": "other",
            "currentState": "BEGIN",
            "masterAddresses": "{}".format(master_address),
            "universeName": self.univ_name,
            "universeUUID": universeUUID,
            "singleStep": "true",
        }

        Output.log("Importing Yugabyte into webserver...")
        try:
            Output.log("Importing master.", logging.DEBUG)
            req = Request(target, headers=headers, data=json.dumps(data).encode('utf8'))
            resp = json.loads(urlopen(req).read())
            checks = resp.get("checks")
            Output.log("Import master payload: req: {}, resp: {}".format(req, resp))
            if universeUUID != resp.get("universeUUID"):
                Output.log(
                    "Failed to import local universe into webserver: invalid uuid: {}".format(resp),
                    logging.ERROR)
            # Node exporter does not matter for local universes and will fail on import.
            if "node_exporter_ip_error_map" in checks:
                del[checks["node_exporter_ip_error_map"]]
            if checks and not all(check == 'OK' for check in checks.values()):
                Output.log(
                    ("Failed to import local universe into webserver, "
                     "checks failed: {}").format(resp))

        except (ValueError, HTTPError, URLError, KeyError) as e:
            return "Failed to import local universe into YW with payload: {}.\n" \
                "Got error: {}".format(data, e)

        Output.log("Import completed.")
        return None

    # Disables/hides paid services on UI.
    def set_landing_page(self, universe_uuid):
        target = "{}/customers/{}/features".format(self.url, self.cust_uuid)
        headers = {
            "X-AUTH-YW-API-TOKEN": self.api_token_secure,
            "Content-Type": "application/json",
        }
        data = {
            "features": {
                "main": {
                    "landing_page": "universes/{}".format(universe_uuid),
                    "universe_list": "disabled"
                }
            }
        }

        err_msg = None
        Output.log("Setting UI landing page...")
        try:
            req = Request(target, headers=headers, data=json.dumps(data).encode('utf8'))
            req.get_method = lambda: "PUT"
            resp = urlopen(req)
            if resp.code != 200:
                err_msg = "Failed setting landing page with error code: {}.".format(resp.code)
        except (ValueError, HTTPError, URLError, KeyError) as e:
            err_msg = "Failed to set landing page: {}".format(e)

        if not err_msg:
            Output.log("Successfully set landing page.")
        else:
            Output.log(err_msg, logging.ERROR)
        return err_msg

    # Sets YugaWare to input security level.
    def set_security(self, level):
        target = "{}/customers/{}/security".format(self.url, self.cust_uuid)
        headers = {
            "X-AUTH-YW-API-TOKEN": self.api_token_secure,
            "Content-Type": "application/json",
        }
        data = {"level": level}
        err_msg = None
        Output.log("Updating YW security to {}...".format(level))
        try:
            req = Request(target, headers=headers, data=json.dumps(data).encode('utf8'))
            req.get_method = lambda: "PUT"
            resp = urlopen(req)
            if resp.code != 200:
                err_msg = "Failed to set security level. YW returned code: " + resp.code
        except (ValueError, HTTPError, URLError, KeyError) as e:
            err_msg = "Failed to set security level: {}".format(e)

        if not err_msg:
            Output.log("Sucesssfully set YW security to {}".format(level))
        else:
            Output.log(err_msg, logging.ERROR)
        return err_msg

    # Add alerts to YugaWare.
    def send_alerts(self, alerts):
        target = "{}/customers/{}/alerts".format(self.url, self.cust_uuid)
        headers = {
            "X-AUTH-YW-API-TOKEN": self.api_token_secure,
            "Content-Type": "application/json",
        }
        Output.log("Adding alerts: {}".format(alerts))
        for alert in alerts:
            data = {
                "type": alert[0],
                "errCode": alert[1],
                "message": alert[2]
            }
            try:
                req = Request(target, headers=headers, data=json.dumps(data).encode('utf8'))
                req.get_method = lambda: "PUT"
                resp = urlopen(req)
                if resp.code != 200:
                    Output.log(
                        "Got error code {} when adding alert: {}".format(resp.code, alert),
                        logging.ERROR)
            except (ValueError, HTTPError, URLError, KeyError) as e:
                Output.log("Failed adding alert {} with error: {}".format(alert, e), logging.ERROR)
        del alerts[:]


# Class that handles any output operations. Use print for what users should see.
# Use log for logging. ANSI escape characters should not be used for logging.
class Output(object):
    supports_color = (sys.platform != 'win32' or 'ANSICON' in os.environ) and \
        hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
    END = "\u001b[0m"
    BOLD = "\u001b[1m"
    UNDERLINE = "\u001b[4m"
    RED = "\u001b[31m"
    GREEN = "\u001b[32m"
    YELLOW = "\u001b[33m"
    BLUE = "\u001b[34m"
    MAGENTA = "\u001b[35m"
    CYAN = "\u001b[36m"
    # Transform to an "extended ASCII" library to parse the string then print it as unicode
    ROCKET = "\xf0\x9f\x9a\x80".encode('latin1').decode('utf8')
    PARTY = "\xf0\x9f\x8e\x89".encode('latin1').decode('utf8')
    SHIRT = "\xf0\x9f\x91\x95".encode('latin1').decode('utf8')
    log_dir = None
    script_exit_func = None
    # Only attempt to write to stdout while we have access to console.
    console_access = True

    ANIMATION_SUCCESS = '\u2705'
    ANIMATION_FAIL = '\u274C'
    ANIMATION_RUNNING = 1
    ANIMATION_WARNING = '\u26A0'
    ANIMATION_STOP = 2
    # Tuple of (animation_status, animation_message) to ensure atomicity.
    animation_status = (ANIMATION_SUCCESS, "")
    animation_thread = None

    @staticmethod
    def print_out(msg):
        if not Output.console_access:
            return

        try:
            try:
                if PY_VERSION < 3:
                    print(msg.encode('utf8'))
                else:
                    print(msg)
            except UnicodeEncodeError:
                print(msg.encode('ascii', 'ignore').decode())
        except Exception as e:
            # Ignore any print errors as they are not critical to the application.
            Output.log("Failed to print with error: {}".format(traceback.format_exc()))

    # Writes one line that may be replaced with the update_animation method.
    # Note - ONLY one line can be replaced. E.g. only characters after a newline can be replaced.
    @staticmethod
    def init_animation(msg):
        if not Output.console_access:
            return

        def animate():
            loading_symbols = ['/', '-', '\\', '|']
            line_len = 0
            i = 0
            running = True
            while running and Output.console_access:
                status, msg = Output.animation_status
                if status == Output.ANIMATION_RUNNING:
                    symbol = loading_symbols[i]
                else:
                    symbol = status
                    running = False

                if msg == "" and status == Output.ANIMATION_STOP:
                    line = "\r\x1b[2K"
                    line_len = 0
                else:
                    line = "\r{} {}".format(symbol, msg)

                line_len = max(len(line), line_len)
                line_to_write = "{:<{}}".format(line, line_len)

                if not running and msg != "":
                    line_to_write += "\n"

                try:
                    try:
                        if PY_VERSION < 3:
                            sys.stdout.write(line_to_write.encode('utf-8'))
                        else:
                            sys.stdout.write(line_to_write)
                    except UnicodeEncodeError:
                        sys.stdout.write(line_to_write.encode('ascii', 'ignore').decode())

                    try:
                        sys.stdout.flush()
                    except IOError as e:
                        Output.log("Errored when flushing stdout: {}".format(e), logging.ERROR)
                    i = (i + 1) % len(loading_symbols)
                except Exception as e:
                    # Ignore stdout write errors as they are not critical to application.
                    Output.log("Failed stdout write with error: {}".format(traceback.format_exc()))

                time.sleep(.05)

        Output.animation_status = (Output.ANIMATION_RUNNING, msg)
        Output.animation_thread = Thread(target=animate)
        Output.animation_thread.start()

    @staticmethod
    def update_animation(msg, status=ANIMATION_SUCCESS):
        if not Output.console_access:
            return

        if not Output.animation_thread:
            Output.print_out(msg)

        Output.animation_status = (status, msg)
        if status != Output.ANIMATION_RUNNING and Output.animation_thread:
            Output.animation_thread.join()
            Output.animation_thread = None

    @staticmethod
    def log(msg, level=logging.INFO):
        if '' == msg or msg is None:
            return

        full_msg = msg
        time_since_sec = time.time() - start_time_sec
        if time_since_sec < 1000:
            # add time since start to make it easier to debug startup perf
            full_msg = " | {:.1f}s | {}".format(time_since_sec, msg)
        try:
            logging.log(level, full_msg)
        except:
            pass

    @staticmethod
    def print_and_log(msg, level=logging.INFO):
        Output.log(msg, level=level)
        Output.print_out(msg)

    @staticmethod
    def log_error_and_exit(msg):
        if Output.log_dir:
            if msg is None:
                msg = "For more information, check the logs in {}".format(Output.log_dir)
            else:
                msg += "\nFor more information, check the logs in {}".format(Output.log_dir)
        Output.print_and_log(msg, logging.ERROR)
        Output.console_access = False
        if Output.script_exit_func:
            Output.script_exit_func()
        sys.exit(1)

    @staticmethod
    def make_bold(msg):
        return Output.BOLD + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_underline(msg):
        return Output.UNDERLINE + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_red(msg):
        return Output.RED + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_green(msg):
        return Output.GREEN + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_yellow(msg):
        return Output.YELLOW + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_blue(msg):
        return Output.BLUE + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_magenta(msg):
        return Output.MAGENTA + msg + Output.END if Output.supports_color else msg

    @staticmethod
    def make_cyan(msg):
        return Output.CYAN + msg + Output.END if Output.supports_color else msg


# Class to customize argparse output.
class PrettyArgParser(argparse.ArgumentParser):
    def __init__(self, **kwargs):
        kwargs["formatter_class"] = PrettyHelpFormatter
        kwargs["epilog"] = EPILOG_COMMON
        super(PrettyArgParser, self).__init__(**kwargs)
        self._positionals.title = Output.make_yellow("Commands")
        self._optionals.title = Output.make_yellow("Flags")

    # Add epilog help message to errors.
    def error(self, message):
        Output.print_out("{} {}.".format(Output.make_red("Error:"), message))
        self.print_help(sys.stderr)
        self.exit(2)


# Class that capitalizes argparse help message.
class PrettyHelpFormatter(argparse.RawTextHelpFormatter):
    # Add prefix of cli title and change "Usage" to yellow
    # Add usage according to the command from which help is called
    def add_usage(self, usage, actions, groups, prefix=None):
        cmd = ('%(prog)s' % dict(prog=self._prog)).split()
        for cmds in ['[command]', '[flags]']:
            if cmds in cmd:
                cmd.remove(cmds)
        if len(cmd) > 2:
            temp = ""
            for cmds in cmd[1:]:
                temp += cmds + " "
            cmd = temp.strip()
        else:
            cmd = cmd.pop()
        if prefix is None:
            prefix = get_cli_title()
            prefix += PREFIX[cmd] + "\n\n"
            prefix += Output.make_yellow('Usage') + ": "
        if usage is None:
            usage = USAGE[cmd]
        super(PrettyHelpFormatter, self).add_usage(
            usage, actions, groups, prefix)

    # Sort the flags in alphabetical order for the help message
    def add_arguments(self, actions):
        actions = sorted(actions, key=operator.attrgetter('option_strings'))
        super(PrettyHelpFormatter, self).add_arguments(actions)

    # Remove the help under "Command" section which had all command inside curly braces
    def _format_action(self, action):
        self._max_help_position = self._action_max_length + 2
        result = super(PrettyHelpFormatter,
                       self)._format_action(action)
        if isinstance(action, argparse._SubParsersAction):
            return "%*s%s" % (self._current_indent, "", result.lstrip())
        return result

    # Remove the help under "Command" section which had all command inside curly braces
    def _iter_indented_subactions(self, action):
        if isinstance(action, argparse._SubParsersAction):
            try:
                get_subactions = action._get_subactions
            except AttributeError:
                pass
            else:
                for subaction in get_subactions():
                    yield subaction
        else:
            for subaction in super(PrettyHelpFormatter,
                                   self)._iter_indented_subactions(action):
                yield subaction

# Returns key-value pairs of input dict. Independent of python version.
def get_kv(map):
    if PY_VERSION < 3:
        return map.iteritems()
    else:
        return map.items()

def run_process(cmd, encrypted_cmd=None, timeout=None, log_cmd=False, env_vars=None, shell=False):
    if log_cmd:
        if encrypted_cmd:
            Output.log("run_process: cmd: {}".format(str(encrypted_cmd)))
        else:
            Output.log("run_process: cmd: {}".format(str(cmd)))

    proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
                            env=env_vars, shell=shell)
    if PY_VERSION >= 3:
        try:
            out, err = proc.communicate(timeout=timeout)
        except subprocess.TimeoutExpired as e:
            if encrypted_cmd:
                Output.log("run_process: {} timeout expired for command: ".format(encrypted_cmd))
            else:
                Output.log("run_process: {} timeout expired for command: ".format(cmd))
            return None, str(e), -1
    else:
        out, err = proc.communicate()
    (ret_out, ret_err, retcode) = (out.decode('utf-8'), err.decode('utf-8'), proc.returncode)
    if log_cmd:
        Output.log("run_process returned {}: \nOUT >>\n{}\n<< ERR >>\n{}\n<<".format(
            retcode, ret_out, ret_err))
    return (ret_out, ret_err, retcode)

def run_process_checked(cmd, timeout=None, log_cmd=True, env_vars=None):
    out, err, retcode = run_process(cmd=cmd, timeout=timeout, log_cmd=log_cmd, env_vars=env_vars)
    if retcode:
        Output.log_error_and_exit("Error: {}".format(err))
    return out

def run_process_with_retries(cmd, encrypted_cmd=None, timeout=None, log_cmd=False, env_vars=None,
                             shell=False, retries=10, sleep_interval=0.2, log_retries=True):
    start_time = time.time()
    now = start_time
    try_count = 0

    # Log command once at the start if log_cmd is True
    if log_cmd:
        if encrypted_cmd:
            Output.log("run_process: cmd: {}".format(str(encrypted_cmd)))
        else:
            Output.log("run_process: cmd: {}".format(str(cmd)))

    while True:
        try_count+=1
        if log_cmd and log_retries:
            Output.log("Running {}. Total retries: {}, Timeout: {}, Try count: {}".format(cmd,
                                                                    retries, timeout, try_count))
        # Only log in run_process if log_retries is True (to avoid logging on each retry)
        out, err, retcode = run_process(cmd=cmd, encrypted_cmd=encrypted_cmd, timeout=timeout,
                                        log_cmd=(log_cmd and log_retries), env_vars=env_vars,
                                        shell=shell)
        now = time.time()
        if retcode:
            if timeout and (now - start_time > timeout):
                return (out, err, retcode)
            elif try_count == retries:
                return (out, err, retcode)
            else:
                time.sleep(sleep_interval)
        else:
            return (out, err, retcode)

def rmcontents(dirname, exclude_names=[]):
    for f in os.listdir(dirname):
        if f in exclude_names:
            continue
        fullpath = os.path.join(dirname, f)
        if os.path.islink(fullpath) or os.path.isfile(fullpath):
            os.unlink(fullpath)
            continue
        if os.path.isdir(fullpath):
            shutil.rmtree(fullpath)
            continue
        Output.log("Unexpected type of file : [ {} ]".format(fullpath))


class RetryableError(Exception):
    pass

# Retry as long as func throws RetryableError.
def retry_op(func, timeout=180):
    start_time = time.time()
    now = start_time
    while True:
        try:
            return func()
        except RetryableError:
            pass
        now = time.time()
        if now - start_time > timeout:
            break
        time.sleep(0.5)

    raise RuntimeError("Failed after retrying operation for {} secs.".format(
        now - start_time))

# Retry the function with argument as long as func throws RetryableError.
def retry_op_with_argument(func, argument, timeout=180):
    start_time = time.time()
    now = start_time
    while True:
        try:
            return func(argument)
        except RetryableError:
            pass
        now = time.time()
        if now - start_time > timeout:
            break
        time.sleep(0.5)

    raise RuntimeError("Failed after retrying operation for {} secs.".format(
        now - start_time))


class EnvBasedCredentials(object):
    _ysql_user = os.environ.get('YSQL_USER')
    _ysql_password = os.environ.get('YSQL_PASSWORD')
    _ysql_db = os.environ.get('YSQL_DB')
    _ycql_user = os.environ.get('YCQL_USER')
    _ycql_password = os.environ.get('YCQL_PASSWORD')
    _ycql_keyspace = os.environ.get('YCQL_KEYSPACE')
    _cert_file_path = os.environ.get('SSL_CERTFILE')

    def attrsetter(attr):
        def set_any(self, value):
            setattr(type(self), attr, value)
        return set_any

    def attrgetter(attr):
        def get_any(self):
            return getattr(self, attr)
        return get_any

    ysql_user = property(attrgetter('_ysql_user'), attrsetter('_ysql_user'))
    ysql_password = property(attrgetter('_ysql_password'), attrsetter('_ysql_password'))
    ysql_db = property(attrgetter('_ysql_db'), attrsetter('_ysql_db'))
    ycql_user = property(attrgetter('_ycql_user'), attrsetter('_ycql_user'))
    ycql_password = property(attrgetter('_ycql_password'), attrsetter('_ycql_password'))
    ycql_keyspace = property(attrgetter('_ycql_keyspace'), attrsetter('_ycql_keyspace'))
    cert_file_path = property(attrgetter('_cert_file_path'), attrsetter('_cert_file_path'))

    def get_ysql_user(self):
        return self.ysql_user or DEFAULT_YSQL_USER

    def get_ysql_password(self):
        return self.ysql_password or DEFAULT_YSQL_PASSWORD

    def get_ysql_db(self):
        return self.ysql_db or DEFAULT_YSQL_DB

    def get_ycql_user(self):
        return self.ycql_user or DEFAULT_YCQL_USER

    def get_ycql_password(self):
        return self.ycql_password or DEFAULT_YCQL_PASSWORD

    def get_ycql_keyspace(self):
        return self.ycql_keyspace or DEFAULT_YCQL_KEYSPACE

    def get_cert_file_path(self):
        return self.cert_file_path

    def get_ysql_credentials(self, get_default_credentials=False):
        if get_default_credentials:
            return DEFAULT_YSQL_USER, DEFAULT_YSQL_PASSWORD, DEFAULT_YSQL_DB
        else:
            return self.get_ysql_user(), self.get_ysql_password(), self.get_ysql_db()

    def get_ycql_credentials(self, get_default_credentials=False):
        if get_default_credentials:
            return DEFAULT_YCQL_USER, DEFAULT_YCQL_PASSWORD, DEFAULT_YCQL_KEYSPACE
        else:
            return self.get_ycql_user(), self.get_ycql_password(), self.get_ycql_keyspace()

    def is_exists(self, var_to_check):
        return True if var_to_check in os.environ and os.environ.get(var_to_check) else False

    def set_ysql_password(self, password):
        self.ysql_password = password

    def set_ycql_password(self, password):
        self.ycql_password = password

    def update_passwords(self, new_password):
        self.set_ysql_password(new_password)
        self.set_ycql_password(new_password)

    def set_ysql_user(self, user):
        self.ysql_user = user

    def set_ysql_db(self, db):
        self.ysql_db = db

    def set_ycql_user(self, user):
        self.ycql_user = user

    def set_ycql_keyspace(self, keyspace):
        self.ycql_keyspace = keyspace

    def setup_ysql_credentials(self, proxy_class):

        # Create DB
        if self.get_ysql_db() != DEFAULT_YSQL_DB and not proxy_class.db_exists(self.get_ysql_db()):
            proxy_class.create_db(self.get_ysql_db())

        # Update password for default user
        if self.get_ysql_user() == DEFAULT_YSQL_USER and self.get_ysql_password() != DEFAULT_YSQL_PASSWORD:
            proxy_class.update_password(self.get_ysql_password())

        # Create User
        if self.get_ysql_user() != DEFAULT_YSQL_USER and not proxy_class.user_exists(self.get_ysql_user()):
            proxy_class.create_user(self.get_ysql_user(), self.get_ysql_password())

            if self.get_ysql_db() != DEFAULT_YSQL_DB:
                proxy_class.db_owner(self.get_ysql_db(), self.get_ysql_user())

            # Note: Following lines will be commented till we can decide on default user deletion.
            #proxy_class.delete_user(self.get_ysql_user(), self.get_ysql_password())

    def setup_ycql_credentials(self, proxy_class):

        # Create YCQL Keyspace
        if self.get_ycql_keyspace() and not proxy_class.keyspace_exists(self.get_ycql_keyspace()):
            proxy_class.create_keyspace(self.get_ycql_keyspace())

        # Update password for default user
        if self.get_ycql_user() == DEFAULT_YCQL_USER and self.get_ycql_password() != DEFAULT_YCQL_PASSWORD:
            proxy_class.update_password(self.get_ycql_password())

        # Create user
        if self.get_ycql_user() != DEFAULT_YCQL_USER and not proxy_class.user_exists(self.get_ycql_user()):
            proxy_class.create_user(self.get_ycql_user(), self.get_ycql_password())

            # Note: Following lines will be commented till we can decide on default user deletion.
            #proxy_class.delete_user(self.get_ycql_user(), self.get_ycql_password())

    def setup_cert_file_path(self, cert_file_path):
        if self.get_cert_file_path() is None or self.get_cert_file_path != cert_file_path:
            os.environ['SSL_CERTFILE'] = cert_file_path
            self.cert_file_path = cert_file_path


if __name__ == '__main__':
    ControlScript().run()
