#!/usr/bin/env python
# Copyright (c) 2020 Klustron inc. All rights reserved.
# This source code is licensed under Apache 2.0 License,
# combined with Common Clause Condition 1.0, as detailed in the NOTICE file.

import sys
import json
import getpass
import argparse
import platform
import os
import copy
from cluster_common import *
from upgrade import *

def purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, usesudo = False):
    process_dirmap(comf, dirmap, machines, usesudo, config)
    process_fileslistmap(comf, filesmap, machines, 'docker', config)
    process_commandslist_setenv(comf, config, machines, commandslist)
    dirmap.clear()
    filesmap.clear()
    del commandslist[:]

def install_docker(args):
    jscfg = get_json_from_file(args.config)
    init_global_docker_config(jscfg, args)
    install_docker_init(jscfg, setup_docker_machines, validate_and_set_docker_config)

def install_docker_init(jscfg, machine_func, validate_func):
    machines = {}
    machine_func(jscfg, machines)
    validate_func(jscfg, machines)
    comf = open(r'docker/install.sh', 'w')
    comf.write('#! /bin/bash\n')
    comf.write("cat /dev/null > runlog\n")
    comf.write("cat /dev/null > lastlog\n")
    if jscfg['config']['verbose']:
        comf.write("trap 'cat lastlog' DEBUG\n")
        comf.write("trap 'exit 1' ERR\n")
    else:
        comf.write("trap 'cat lastlog >> runlog' DEBUG\n")
        comf.write("trap 'cat lastlog; exit 1' ERR\n")
    install_with_config(jscfg, comf, machines)
    output_info(comf, "Installation completed !")
    comf.close()

def clean_docker(args):
    jscfg = get_json_from_file(args.config)
    init_global_docker_config(jscfg, args)
    clean_docker_init(jscfg, setup_docker_machines, validate_and_set_docker_config)

def clean_docker_init(jscfg, machine_func, validate_func):
    machines = {}
    machine_func(jscfg, machines)
    validate_func(jscfg, machines)
    comf = open(r'docker/clean.sh', 'w')
    comf.write('#! /bin/bash\n')
    comf.write("cat /dev/null > runlog\n")
    comf.write("cat /dev/null > lastlog\n")
    if jscfg['config']['verbose']:
        comf.write("trap 'cat lastlog' DEBUG\n")
    else:
        comf.write("trap 'cat lastlog >> runlog' DEBUG\n")
    clean_with_config(jscfg, comf, machines)
    output_info(comf, "Clean action completed !")
    comf.close()

def start_docker(args):
    jscfg = get_json_from_file(args.config)
    init_global_docker_config(jscfg, args)
    start_docker_init(jscfg, setup_docker_machines, validate_and_set_docker_config)

def start_docker_init(jscfg, machine_func, validate_func):
    machines = {}
    machine_func(jscfg, machines)
    validate_func(jscfg, machines)
    comf = open(r'docker/start.sh', 'w')
    comf.write('#! /bin/bash\n')
    comf.write("cat /dev/null > runlog\n")
    comf.write("cat /dev/null > lastlog\n")
    if jscfg['config']['verbose']:
        comf.write("trap 'cat lastlog' DEBUG\n")
        comf.write("trap 'exit 1' ERR\n")
    else:
        comf.write("trap 'cat lastlog >> runlog' DEBUG\n")
        comf.write("trap 'cat lastlog; exit 1' ERR\n")
    start_with_config(jscfg, comf, machines)
    output_info(comf, "Start action completed !")
    comf.close()

def stop_docker(args):
    jscfg = get_json_from_file(args.config)
    init_global_docker_config(jscfg, args)
    stop_docker_init(jscfg, setup_docker_machines, validate_and_set_docker_config)

def stop_docker_init(jscfg, machine_func, validate_func):
    machines = {}
    machine_func(jscfg, machines)
    validate_func(jscfg, machines)
    comf = open(r'docker/stop.sh', 'w')
    comf.write('#! /bin/bash\n')
    comf.write("cat /dev/null > runlog\n")
    comf.write("cat /dev/null > lastlog\n")
    if jscfg['config']['verbose']:
        comf.write("trap 'cat lastlog' DEBUG\n")
    else:
        comf.write("trap 'cat lastlog >> runlog' DEBUG\n")
    stop_with_config(jscfg, comf, machines)
    output_info(comf, "Stop action completed !")
    comf.close()

def generate_mount_param(node, dirattrlist):
    param = " "
    for item in dirattrlist:
        attrname = item[0]
        defdir = item[1]
        if attrname in node and node[attrname] is not None:
            param = " %s -v %s:%s " % (param, node[attrname], defdir)
    return param

def setup_onexpanel(jscfg, machines, comf, config, xpanel, node, image, imageFile):
    mach = machines.get(node['ip'])
    if xpanel['imageType'] == 'file':
        if not config['cloud']:
            process_command_noenv(comf, config, machines, node['ip'], '/',
                    'sudo mkdir -p %s && sudo chown -R %s:\`id -gn %s\` %s' % (mach['basedir'],
                    mach['user'], mach['user'], mach['basedir']))
            process_file(comf, config, machines, node['ip'], 'docker/%s' % imageFile, mach['basedir'])
        cmdpat = "sudo docker inspect %s >& /dev/null || ( gzip -cd %s | sudo docker load )"
        mach = machines.get(node['ip'])
        process_command_noenv(comf, config, machines, node['ip'], mach['basedir'], 
                    cmdpat % (image, imageFile))

def setup_xpanel(jscfg, machines, comf):
    if 'xpanel' not in jscfg:
        return
    config = jscfg['config']
    xpanel = jscfg['xpanel']
    for node in xpanel['nodes']:
        output_info(comf, "Setting up xpanel on %s ..." % node['ip'])
        setup_onexpanel(jscfg, machines, comf, config, xpanel, node, xpanel['image'], xpanel['imageFile'])

def install_xpanel(jscfg, machines, dirmap, filesmap, commandslist, metaseeds, comf):
    if 'xpanel' not in jscfg:
        return
    config = jscfg['config']
    autostart = config['autostart']
    xpanel = jscfg['xpanel']
    network = config['network']
    namespace = config['namespace']
    restart = 'no'
    if autostart:
        restart = 'always'
    for node in xpanel['nodes']:
        mach = machines.get(node['ip'])
        output_info(comf, "Starting xpanel on %s ..." % node['ip'])
        cmdpat = "sudo docker run --network %s --privileged -itd --restart={} --env METASEEDS=%s --name %s -p %d:80 %s bash -c '/bin/bash /kunlun/start.sh'".format(restart)
        process_command_noenv(comf, config, machines, node['ip'], '/', cmdpat % (network, metaseeds, node['name'], node['port'], xpanel['image']))

def setup_storage_node(config, nodemgrobjs, node, dirattrlist, storagetemplate, storageurl, dockercmd,
        metaseeds, masterip, cluster_name, uuid, iplist, idx, hamode, cmdlist, waitlist):
    if hamode == 'mgr':
        cmdpat= dockercmd + " %s --network %s --name %s -h %s %s %s /bin/bash start_storage_mgr.sh %s %s %s %d %s %s %s %s"
    elif hamode == 'rbr':
        cmdpat= dockercmd + " %s --network %s --name %s -h %s %s %s /bin/bash start_storage_rbr.sh %s %s %s %d %s %s %s"
    else:
        # no_rep
        cmdpat= dockercmd + " %s --network %s --name %s -h %s %s %s /bin/bash start_storage_norep.sh %s %d %s %s"
    if config['small']:
        cmdpat += ' small'
    network = config['network']
    waitcmdpat="sudo docker exec %s /bin/bash /kunlun/wait_storage_up.sh"
    nodemgrobj = copy.deepcopy(storagetemplate)
    nodemgrobj['ip'] = node['ip']
    nodemgrobj['hostip'] = node['hostip']
    nodemgrobj['nodetype'] = 'storage'
    nodemgrobjs.append(nodemgrobj)
    targetdir="."
    buf=node['buf']
    orig = node['orig']
    mountarg=generate_mount_param(node, dirattrlist)
    if node['is_primary']:
        if hamode == 'mgr':
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl, uuid,
                    node['ip'], iplist, idx, str(node['is_primary']).lower(), buf, cluster_name, node['shard_id']))
        elif hamode == 'no_rep':
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl, 
                    buf, idx, cluster_name, node['shard_id']))
        else:
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl,
                    metaseeds, node['ip'], masterip, idx, buf, cluster_name, node['shard_id']))
        addToCommandsList(waitlist, node['hostip'], targetdir,	waitcmdpat % (node['ip']))
    else:
        if hamode == 'mgr':
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl, uuid,
                    node['ip'], iplist, idx, str(node['is_primary']).lower(), buf, cluster_name, node['shard_id']))
        elif hamode == 'no_rep':
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl,
                    buf, cluster_name, node['shard_id']))
        else:
            addToCommandsList(cmdlist, node['hostip'], targetdir,
                    cmdpat % (node['dockeropts'], network, node['ip'], node['ip'], mountarg, storageurl,
                    metaseeds, node['ip'], masterip, idx, buf, cluster_name, node['shard_id']))
        addToCommandsList(waitlist, node['hostip'], targetdir, waitcmdpat % (node['ip']))
    del node['hostip']
    del node['buf']
    del node['orig']
    del node['dockeropts']
    del node['shard_id']

def setup_server_node(config, nodemgrobjs, node, dirattrlist, servertemplate, serverurl, dockercmd,
       metaseeds, i, ha_mode, cmdlist, waitlist):
    if ha_mode == 'rbr':
        cmdpat= dockercmd + r' %s --network %s --name %s -h %s -p %d:5432 %s %s /bin/bash start_server_rbr.sh %s %s %s "%s" %d'
    else:
        cmdpat= dockercmd + r' %s --network %s --name %s -h %s -p %d:5432 %s %s /bin/bash start_server.sh %s "%s" %d'
    waitcmdpat="sudo docker exec %s /bin/bash /kunlun/wait_server_up.sh"
    waitlist = []
    network = config['network']
    targetdir = "."
    mountarg=generate_mount_param(node, dirattrlist)
    name = node['ip']
    nodemgrobj = copy.deepcopy(servertemplate)
    nodemgrobj['ip'] = node['ip']
    nodemgrobj['hostip'] = node['hostip']
    nodemgrobj['nodetype'] = 'server'
    nodemgrobjs.append(nodemgrobj)
    dockeropts = node.get('dockeropts', config['dockeropts'])
    if ha_mode == 'rbr':
        addToCommandsList(cmdlist, node['hostip'], targetdir,
              cmdpat % (dockeropts, network, name, name, node['port'], mountarg, serverurl,
              metaseeds, name, node['user'], node['password'], i))
    else:
        addToCommandsList(cmdlist, node['hostip'], targetdir,
              cmdpat % (dockeropts, network, name, name, node['port'], mountarg, serverurl,
              node['user'], node['password'], i))
    addToCommandsList(waitlist, node['hostip'], targetdir,  waitcmdpat % (name))

def install_clusters1(jscfg, machines, dirmap, filesmap, commandslist, metaseeds, comf, 
        storageurl, serverurl, dockercmd, storagetemplate, servertemplate, nodemgrobjs, 
        metafile, metadataobj, meta_hamode):
    if 'clusters1' not in jscfg:
        return
    storage_dirattrlist = [
            ["storage_datadir", "/kunlun/storage_datadir"],
            ["storage_logdir", "/kunlun/storage_logdir"],
            ["storage_waldir", "/kunlun/storage_waldir"],
            ["storage_keyringdir", "/kunlun/storage_keyringdir"],
            ["prometheus_datadir", "/kunlun/prometheus_datadir"]
        ]
    server_dirattrlist = [
            ["server_datadir", "/kunlun/server_datadir"],
            ["prometheus_datadir", "/kunlun/prometheus_datadir"]
        ]
    clusters = jscfg['clusters1']
    config = jscfg['config']
    defbufstr='1024MB'
    elements = metadataobj['nodemapmaster']['elements']
    for cluster in clusters:
        output_info(comf, "installing cluster %s ..." % cluster['name'])
        cluster_name = cluster['name']
        ha_mode = cluster['ha_mode']
        pricmdlist = []
        secmdlist = []
        priwaitlist = []
        sewaitlist = []
        datanodes = []
        for shard in cluster['data']:
            nodes = []
            nodelist = []
            shardname = None
            for node in shard['nodes']:
                #my_print('installing %s' % node['name'])
                if shardname is None:
                    shardname = node['shard_id']
                bufsize=node.get('innodb_buffer_pool_size', "")
                nodeobj={"port":3306, "user":"pgx", "password":"pgx_pwd", "ip":node['name'],
                        "hostip":node['ip'], "is_primary":node.get('is_primary', False),
                        "dockeropts": node.get('dockeropts', config['dockeropts']), 
                        "buf":node.get('innodb_buffer_pool_size', defbufstr), "orig":node,
                        "shard_id": node['shard_id']}
                nodelist.append(node['name'])
                nodes.append(nodeobj)
            iplist=",".join(nodelist)
            shard_obj={"shard_name":shardname, "shard_nodes":nodes}
            datanodes.append(shard_obj)
            uuid=getuuid()
            masterip = get_masterip(nodes)
            pnode = None
            snodes = []
            j = 1
            for node in nodes:
                if node['is_primary']:
                    pnode = node
                    setup_storage_node(config, nodemgrobjs, node, storage_dirattrlist, storagetemplate, storageurl, dockercmd,
                            metaseeds, masterip, cluster_name, uuid, iplist, j, ha_mode, pricmdlist, priwaitlist)
                else:
                    snodes.append(node)
                    setup_storage_node(config, nodemgrobjs, node, storage_dirattrlist, storagetemplate, storageurl, dockercmd,
                            metaseeds, masterip, cluster_name, uuid, iplist, j, ha_mode, secmdlist, sewaitlist)
                j+=1
            if ha_mode == 'rbr':
                for node in snodes:
                    elements.append({"host": node['ip'], "port":node['port'], 
                        "master_host":pnode['ip'], "master_port":pnode['port'], "is_meta": False})
        commandslist.extend(pricmdlist)
        commandslist.extend(secmdlist)
        commandslist.extend(priwaitlist)
        commandslist.extend(sewaitlist)
        cmdlist = []
        waitlist = []
        compnodes = []
        i = 1
        for node in cluster['comp']['nodes']:
            #my_print("installing %s" % node['name'])
            comp={"id":i, "user":node['user'], "password":node['password'],
                    'hostip':node['ip'], "name":node['name'], "ip":node['name'], "port":5432, "mysql_port":3306}
            setup_server_node(config, nodemgrobjs, comp, server_dirattrlist, servertemplate, serverurl, dockercmd,
                    metaseeds, i, ha_mode, cmdlist, waitlist)
            del comp['hostip']
            compnodes.append(comp)
            i += 1
        commandslist.extend(cmdlist)
        commandslist.extend(waitlist)
        dataname = "%s_shards.json" % cluster_name
        dataf = open('docker/%s' % dataname, 'w')
        json.dump(datanodes, dataf, indent=4)
        dataf.close()
        compname = "%s_comps.json" % cluster_name
        compf = open('docker/%s' % compname, 'w')
        json.dump(compnodes, compf, indent=4)
        compf.close()
        firstcomp = cluster['comp']['nodes'][0]
        addIpToFilesListMap(filesmap, firstcomp['ip'], dataname, '.')
        addIpToFilesListMap(filesmap, firstcomp['ip'], compname, '.')
        cmdpat = "sudo docker cp %s %s:/kunlun"
        addToCommandsList(commandslist, firstcomp['ip'], '.', cmdpat % (metafile, firstcomp['name']))
        addToCommandsList(commandslist, firstcomp['ip'], '.', cmdpat % (dataname, firstcomp['name']))
        addToCommandsList(commandslist, firstcomp['ip'], '.', cmdpat % (compname, firstcomp['name']))
        addToCommandsList(commandslist, firstcomp['ip'], '.', 'sleep 60')
        cmdpat = "sudo docker exec %s /bin/bash /kunlun/create_cluster.sh %s %s %s %s %s %s"
        addToCommandsList(commandslist, firstcomp['ip'], '.', cmdpat % (firstcomp['name'], cluster_name,
            meta_hamode, ha_mode, '/kunlun/%s' % metafile, '/kunlun/%s' % dataname, '/kunlun/%s' % compname))

def stop_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf):
    if 'clusters1' not in jscfg:
        return
    cmdpat = 'sudo docker container stop %s'
    for cluster in jscfg['clusters1']:
        output_info(comf, "stopping cluster %s ..." % cluster['name'])
        for node in cluster['comp']['nodes']:
            addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])
        for shard in cluster['data']:
            for node in shard['nodes']:
                addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])

def start_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf):
    if 'clusters1' not in jscfg:
        return
    cmdpat = 'sudo docker container start %s'
    for cluster in jscfg['clusters1']:
        output_info(comf, "starting cluster %s ..." % cluster['name'])
        for node in cluster['comp']['nodes']:
            addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])
        for shard in cluster['data']:
            for node in shard['nodes']:
                addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])

def clean_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf):
    if 'clusters1' not in jscfg:
        return
    cmdpat = 'sudo docker container rm -f %s'
    for cluster in jscfg['clusters1']:
        output_info(comf, "cleaning cluster %s ..." % cluster['name'])
        for node in cluster['comp']['nodes']:
            addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])
        for shard in cluster['data']:
            for node in shard['nodes']:
                addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['name'])

def install_with_config(jscfg, comf, machines):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    config = jscfg['config']
    meta_hamode = meta.get('ha_mode', '')
    version = config['product_version']
    dirmap = {}
    filesmap = {}
    commandslist = []

    dirattrlist = [
            ["storage_datadir", "/kunlun/storage_datadir"],
            ["storage_logdir", "/kunlun/storage_logdir"],
            ["storage_waldir", "/kunlun/storage_waldir"],
            ["server_datadir", "/kunlun/server_datadir"],
            ["storage_keyringdir", "/kunlun/storage_keyringdir"],
            ["prometheus_datadir", "/kunlun/prometheus_datadir"]
        ]

    storageurl = ""
    serverurl = ""
    clustermgrurl = ""
    nodeurl = ""
    if config['imageInFiles']:
        storageurl = 'kunlun-storage:%s' % version
        serverurl = 'kunlun-server:%s' % version
        clustermgrurl = 'kunlun-cluster-manager:%s' % version
        nodeurl = 'kunlun-node:%s' % version
        dockercmd = "sudo docker run --privileged -itd "
    else:
        images = jscfg['images']
        storageurl = "%s:%s" % (images['kunlun-storage'], version)
        serverurl = "%s:%s" % (images['kunlun-server'], version)
        clustermgrurl = "%s:%s" % (images['kunlun-cluster-manager'], version)
        nodeurl = "%s:%s" % (images['kunlun-node'], version)
        dockercmd = "sudo docker run --privileged --pull always -itd "
    if config['autostart']:
        dockercmd = "%s --restart=always " % dockercmd
    network = config['network']
    namespace = config['namespace']
    meta_ha_mode = meta['ha_mode']
    defbufstr='1024MB'

    metadataobj = {}
    nodemgrobjs = []
    storage_template = {
                "total_cpu_cores": 3,
                "total_mem": 8192,
                "storage_usedports": ["3306", "33060", "33062"],
                "server_usedports": [],
                "skip": False,
                "storage_portrange": "57000-58000", 
                "server_portrange": "47000-48000", 
                "storage_curport": 57001, 
                "server_curport": 47001, 
                "prometheus_port_start": 58010, 
                "brpc_http_port": 35001, 
                "tcp_port": 35002, 
                "server_datadirs": "/kunlun/server_datadir", 
                "storage_datadirs": "/kunlun/storage_datadir", 
                "storage_logdirs": "/kunlun/storage_logdir", 
                "storage_waldirs": "/kunlun/storage_waldir",
                "storage_keyringdir": "/kunlun/storage_keyringdir"
                }
    server_template = {
                "total_cpu_cores": 3,
                "total_mem": 8192,
                "storage_usedports": [],
                "server_usedports": ["3306", "5432"],
                "skip": False,
                "storage_portrange": "57000-58000", 
                "server_portrange": "47000-48000", 
                "storage_curport": 57001, 
                "server_curport": 47001, 
                "prometheus_port_start": 58010, 
                "brpc_http_port": 35001, 
                "tcp_port": 35002, 
                "server_datadirs": "/kunlun/server_datadir" 
                }
    # Meta nodes
    metanodes = []
    metalist=[]
    meta_addrs = []
    for node in meta['nodes']:
        meta_addrs.append(node['uri'])
        metaobj={"port":3306, "user":"pgx", "password":"pgx_pwd", "ip":node['name'],
		  "hostip":node['ip'], "is_primary":node.get('is_primary', False),
		  "buf":node.get('innodb_buffer_pool_size', defbufstr), "orig":node,
                "dockeropts": node.get('dockeropts', config['dockeropts']), 
                "shard_id": node['shard_id']}
        metalist.append(node['name'])
        metanodes.append(metaobj)
    iplist=",".join(metalist)
    metaseeds = ",".join(meta_addrs)
    output_info(comf, "install meta nodes ...")
    uuid=getuuid()
    secmdlist=[]
    priwaitlist=[]
    sewaitlist=[]
    masterip = get_masterip(metanodes)
    cluster_name = "meta_cluster"
    i = 1
    pnode = None
    snodes = []
    if 'nodemapmaster' not in metadataobj:
        metadataobj['nodemapmaster'] = {'op':'add', "elements":[]}
    elements = metadataobj['nodemapmaster']["elements"]
    for node in metanodes:
        if node['is_primary']:
            pnode = node
            setup_storage_node(config, nodemgrobjs, node, dirattrlist, storage_template, storageurl, dockercmd,
                metaseeds, masterip, cluster_name, uuid, iplist, i, meta_hamode, commandslist, priwaitlist)
        else:
            snodes.append(node)
            setup_storage_node(config, nodemgrobjs, node, dirattrlist, storage_template, storageurl, dockercmd,
                metaseeds, masterip, cluster_name, uuid, iplist, i, meta_hamode, secmdlist, sewaitlist)
        i+=1
    if meta_hamode == 'rbr':
        for node in snodes:
            # my_print("add nodemap_master for %s" % node['ip'])
            elements.append({"host": node['ip'], "port":node['port'], 
                "master_host":pnode['ip'], "master_port":pnode['port'], "is_meta": True})
    commandslist.extend(priwaitlist)
    commandslist.extend(secmdlist)
    commandslist.extend(sewaitlist)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    targetdir="."
    metafile = None
    if len(metanodes) > 0:
        firstmeta = meta['nodes'][0]
        #my_print(str(firstmeta))
        metafile = 'docker-meta.json'
        metaf = open("docker/%s" % metafile, 'w')
        json.dump(metanodes, metaf, indent=4)
        metaf.close()
        addIpToFilesListMap(filesmap, firstmeta['ip'], metafile, '.')
        addIpToFilesListMap(filesmap, firstmeta['ip'], "../clustermgr/dba_tools_db.sql", '.')
        cmdpat = "sudo docker cp %s %s:/kunlun"
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % (metafile, firstmeta['name']))
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % ('dba_tools_db.sql', firstmeta['name']))
        cmdpat = "sudo docker exec %s /bin/bash init_meta.sh %s %s"
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % (firstmeta['name'], metafile, meta_ha_mode))
        cmdpat = "sudo docker exec %s /bin/bash init_xpaneldb.sh"
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % firstmeta['name'])
        purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    nodemgr_template = {
                "total_cpu_cores": 3,
                "total_mem": 8192,
                "storage_usedports": [],
                "server_usedports": [],
                "skip": False,
                "storage_portrange": "57000-58000", 
                "server_portrange": "47000-48000", 
                "storage_curport": 57001, 
                "server_curport": 47001, 
                "prometheus_port_start": 58010, 
                "brpc_http_port": 35001, 
                "tcp_port": 35002, 
                "server_datadirs": "/kunlun/server_datadir", 
                "storage_datadirs": "/kunlun/storage_datadir", 
                "storage_logdirs": "/kunlun/storage_logdir", 
                "storage_waldirs": "/kunlun/storage_waldir",
                "storage_keyringdir": "/kunlun/storage_keyringdir"
                }
    # nodemgr
    cmdpat= dockercmd + " %s --network %s --name %s -h %s %s /bin/bash /kunlun/start_node.sh %s %s"
    for node in nodemgr['nodes']:
        nodemgrobj = copy.deepcopy(nodemgr_template)
        nodemgrobj['ip'] = node['name']
        nodemgrobj['hostip'] = node['ip']
        nodemgrobj['nodetype'] = node.get('nodetype', 'both')
        nodemgrobjs.append(nodemgrobj)
        name = node['name']
        output_info(comf, "start node_manager(%s) container on %s ..." % (node['name'], node['ip']))
        dockeropts = node.get('dockeropts', config['dockeropts'])
        addToCommandsList(commandslist, node['ip'], targetdir,
	    cmdpat % (dockeropts, network, name, name, nodeurl, metaseeds, name))
        purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    # install clusters1
    install_clusters1(jscfg, machines, dirmap, filesmap, commandslist, metaseeds, comf,
            storageurl, serverurl, dockercmd, storage_template, server_template, nodemgrobjs,
            metafile, metadataobj, meta_ha_mode)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    metadataobj['node_manager'] = {'op':'add', 'elements': nodemgrobjs}
    metadatajson = 'metadata.json'
    metadataf = open('docker/%s' % metadatajson, 'w')
    json.dump(metadataobj, metadataf, indent=4)
    metadataf.close()
    targetdir = "."
    firstnode = nodemgrobjs[0]
    addIpToFilesListMap(filesmap, firstnode['hostip'], metadatajson,  targetdir)
    addIpToFilesListMap(filesmap, firstnode['hostip'], "../clustermgr/modify_metadata.py", targetdir)
    cmdpat = "sudo docker cp %s %s:/kunlun"
    addToCommandsList(commandslist, firstnode['hostip'], targetdir, cmdpat % (metadatajson, firstnode['ip']))
    addToCommandsList(commandslist, firstnode['hostip'], targetdir, cmdpat % ('modify_metadata.py', firstnode['ip']))
    cmdpat = "sudo docker exec %s /usr/bin/python2 /kunlun/modify_metadata.py --config=%s --seeds=%s"
    addToCommandsList(commandslist, firstnode['hostip'], targetdir, cmdpat % (firstnode['ip'], metadatajson, metaseeds))
        
    # clustermgr
    host_str = ",".join(clustermgr['hosts'])
    cmdpat= dockercmd + " %s --network %s --name %s -h %s %s /bin/bash /kunlun/start_cluster_manager.sh %s %s %s"
    for node in clustermgr['nodes']:
        output_info(comf, "start cluster_manager(%s) container on %s ..." % (node['name'], node['ip']))
        name = node['name']
        dockeropts = node.get('dockeropts', config['dockeropts'])
        addToCommandsList(commandslist, node['ip'], targetdir,
	    cmdpat % (dockeropts, network, name, name, clustermgrurl, metaseeds, name, host_str))
        purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    # install xpanel
    install_xpanel(jscfg, machines, dirmap, filesmap, commandslist, metaseeds, comf)

def clean_onexpanel(jscfg, machines, comf, config, xpanel, node, image, imageFile):
    cmdpat = "sudo docker container rm -f %s"
    process_command_noenv(comf, config, machines, node['ip'], '/', cmdpat % node['name'])
    cmdpat = "sudo docker image rm -f %s"
    process_command_noenv(comf, config, machines, node['ip'], '/', cmdpat % image)
    if xpanel['imageType'] == 'file' and not config['cloud']:
        mach = machines.get(node['ip'])
        process_command_noenv(comf, config, machines, node['ip'], mach['basedir'], 'rm -f %s' % imageFile)

def clean_xpanel(jscfg, machines, comf):
    if 'xpanel' not in jscfg:
        return
    config = jscfg['config']
    xpanel = jscfg['xpanel']
    for node in xpanel['nodes']:
        output_info(comf, "Cleaning xpanel on %s ..." % node['ip'])
        clean_onexpanel(jscfg, machines, comf, config, xpanel, node, xpanel['image'], xpanel['imageFile'])

def start_xpanel(jscfg, machines, dirmap, filesmap, commandslist, comf):
    if 'xpanel' not in jscfg:
        return
    config = jscfg['config']
    xpanel = jscfg['xpanel']
    for node in xpanel['nodes']:
        output_info(comf, "Starting xpanel on %s ..." % node['ip'])
        cmdpat = "sudo docker container start %s"
        process_command_noenv(comf, config, machines, node['ip'], '/', cmdpat % node['name'])

def stop_xpanel(jscfg, machines, dirmap, filesmap, commandslist, comf):
    if 'xpanel' not in jscfg:
        return
    config = jscfg['config']
    xpanel = jscfg['xpanel']
    for node in xpanel['nodes']:
        output_info(comf, "Stopping xpanel on %s ..." % node['ip'])
        cmdpat = "sudo docker container stop %s"
        process_command_noenv(comf, config, machines, node['ip'], '/', cmdpat % node['name'])

def clean_with_config(jscfg, comf, machines):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    config = jscfg['config']
    version = config['product_version']
    dirmap = {}
    filesmap = {}
    commandslist = []

    storageurl = ""
    serverurl = ""
    clustermgrurl = ""
    nodeurl = ""
    if config['imageInFiles']:
        storageurl = 'kunlun-storage:%s' % version
        serverurl = 'kunlun-server:%s' % version
        clustermgrurl = 'kunlun-cluster-manager:%s' % version
        nodeurl = 'kunlun-node:%s' % version
    else:
        images = jscfg['images']
        storageurl = "%s:%s" % (images['kunlun-storage'], version)
        serverurl = "%s:%s" % (images['kunlun-server'], version)
        clustermgrurl = "%s:%s" % (images['kunlun-cluster-manager'], version)
        nodeurl = "%s:%s" % (images['kunlun-node'], version)
    network = config['network']
    namespace = config['namespace']

    cmdpat= "sudo docker container stop %s; sudo docker container rm -v %s"
    vcmdpat = "sudo docker image rm -f %s || true"
    rmcmdpat = "sudo rm -fr %s"
    targetdir = "."

    output_info(comf, "clean the meta nodes ...")
    metaips = set()
    # remove meta nodes:
    for node in meta['nodes']:
        name = node['name']
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (name, name))
        metaips.add(node['ip'])
        # TODO: remove the volume mount point
    for ip in metaips:
        addToCommandsList(commandslist, ip, targetdir, vcmdpat % storageurl)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    clean_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "clean the cluster_manager nodes ...")
    clustermgrips = set()
    # clustermgr
    for node in clustermgr['nodes']:
        name = node['name']
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (name, name))
        clustermgrips.add(node['ip'])
    for ip in clustermgrips:
        addToCommandsList(commandslist, ip, targetdir, vcmdpat % clustermgrurl)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "clean the node_manager nodes ...")
    nodemgrips = set()
    # clustermgr
    for node in nodemgr['nodes']:
        name = node['name']
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % (name, name))
        nodemgrips.add(node['ip'])
    for ip in nodemgrips:
        addToCommandsList(commandslist, ip, targetdir, vcmdpat % nodeurl)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    # clean xpanel
    clean_xpanel(jscfg, machines, comf)

def start_with_config(jscfg, comf, machines):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    config = jscfg['config']
    dirmap = {}
    filesmap = {}
    commandslist = []

    cmdpat= "sudo docker container start %s"
    targetdir = "."

    output_info(comf, "start the meta nodes ...")
    # remove meta nodes:
    for node in meta['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    start_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "start the cluster_manager nodes ...")
    # clustermgr
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "start the node_manager nodes ...")
    # clustermgr
    for node in nodemgr['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    # stop xpanel
    start_xpanel(jscfg, machines, dirmap, filesmap, commandslist, comf)

def stop_with_config(jscfg, comf, machines):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    config = jscfg['config']
    dirmap = {}
    filesmap = {}
    commandslist = []

    cmdpat= "sudo docker container stop %s"
    targetdir = "."

    output_info(comf, "stop the meta nodes ...")
    # remove meta nodes:
    for node in meta['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "stop the cluster_manager nodes ...")
    # clustermgr
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    stop_clusters1(jscfg, machines, dirmap, filesmap, commandslist, comf)
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    output_info(comf, "stop the node_manager nodes ...")
    # clustermgr
    for node in nodemgr['nodes']:
        addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['name'])
    purge_cache_commands(config, comf, machines, dirmap, filesmap, commandslist, True)

    # stop xpanel
    stop_xpanel(jscfg, machines, dirmap, filesmap, commandslist, comf)

def get_x86_64_3rdpackages_filemap():
    return {
            "haproxy": ["haproxy-2.5.0-bin.tar.gz", "haproxy-2.5.0-bin"]
            }

def get_aarch64_3rdpackages_filemap():
    return {
            "haproxy": ["haproxy-2.5.0-bin.tar.gz", "haproxy-2.5.0-bin"]
            }

def get_arch_3rdpackages_filemap(config):
    arch = config['targetarch']
    if arch == 'x86_64':
        return get_x86_64_3rdpackages_filemap()
    elif arch == 'aarch64':
        return get_aarch64_3rdpackages_filemap()
    else : # not ready for loongarch64, etc
        raise ValueError('bad arch: %s' % arch)

def get_3rdpackages_filemap(config):
    return get_arch_3rdpackages_filemap(config)

def download_docker_packages(args):
    if args.config == '':
        jscfg = {}
    else:
        jscfg = get_json_from_file(args.config)
    init_global_docker_config(jscfg, args)
    config = jscfg['config']
    arch = config['targetarch']
    prodver = config['product_version']
    downtype = config['downloadtype']
    contentTypes = set()
    downbase = get_downloadbase(config['downloadsite'])
    targetdir="docker"
    contentTypes.add('application/x-gzip')
    imgnames = ["kunlun-storage", "kunlun-server", "kunlun-cluster-manager", "kunlun-node"]
    # download the binary packages
    for name in imgnames:
        fname = "%s-%s.tar.gz" % (name, prodver)
        if downtype == 'release':
            fpath = "releases_%s/%s/docker-multi/%s" % (arch, prodver, fname)
        else:
            fpath = "dailybuilds_%s/docker-images/%s" % (arch, fname)
        download_file(downbase, fpath, contentTypes, targetdir, args.overwrite)
    archmap = get_arch_3rdpackages_filemap(config)
    for pkgname in archmap:
        finfo = archmap[pkgname]
        fpath = 'contrib/%s/%s' % (arch, finfo[0])
        download_file(downbase, fpath, contentTypes, targetdir, args.overwrite)

if  __name__ == '__main__':
    defconfig = gen_default_docker_config()
    parser = argparse.ArgumentParser(description='Specify the arguments.')
    actions=["download", "install", "start", "stop", "clean"]
    parser.add_argument('--action', type=str, help="The action", required=True, choices=actions, default="install")
    parser.add_argument('--config', type=str, help="The config path", default="")
    parser.add_argument('--defuser', type=str, help="the command", default=defconfig['defuser'])
    parser.add_argument('--defbase', type=str, help="the command", default=defconfig['defbase'])
    parser.add_argument('--autostart', help="whether to start the container always", default=defconfig['autostart'], action='store_true')
    parser.add_argument('--small', help="whether to use small template", default=defconfig['small'], action='store_true')
    parser.add_argument('--verbose', help="verbose mode, to show more information",
            default=defconfig['verbose'], action='store_true')
    parser.add_argument('--network', type=str, help="the default network", default=defconfig['network'])
    parser.add_argument('--namespace', type=str, help="the default namespace", default=defconfig['namespace'])
    parser.add_argument('--localip', type=str, help="The local ip address", default=defconfig['localip'])
    parser.add_argument('--product_version', type=str, help="kunlun version", default=defconfig['product_version'])
    parser.add_argument('--dockeropts', type=str, help="the default docker options", default=defconfig['dockeropts'])
    parser.add_argument('--imageInFiles', help="whether to use image in files", default=defconfig['imageInFiles'], action='store_true')
    parser.add_argument('--download', help="whether to overwrite existing file during download", default=defconfig['download'], action='store_true')
    parser.add_argument('--downloadsite', type=str, help="the download base site",
            choices=['public', 'devsite', 'internal'], default=defconfig['downloadsite'])
    parser.add_argument('--downloadtype', type=str, help="the packages type", 
            choices=['release', 'daily_rel', 'daily_debug'], default=defconfig['downloadtype'])
    parser.add_argument('--targetarch', type=str, help="the cpu arch for the packages to download/install",
            default=defconfig['targetarch'])
    parser.add_argument('--overwrite', help="whether to overwrite existing file during download",
            default=defconfig['overwrite'], action='store_true')

    args = parser.parse_args()
    if not args.defbase.startswith('/'):
        raise ValueError('Error: the default basedir must be absolute path!')

    # set the default config file to be config.json for non-download action.
    if args.config == '' and args.action != 'download':
        args.config = 'config_docker.json'

    my_print(str(sys.argv))
    checkdirs(['docker'])

    if args.action == 'download':
        download_docker_packages(args)
    elif args.action == 'install':
        if args.download:
            download_docker_packages(args)
        install_docker(args)
    elif args.action == 'clean':
        clean_docker(args)
    elif args.action == 'start':
        start_docker(args)
    elif args.action == 'stop':
        stop_docker(args)
    else:
        # just defensive, for more more actions later.
        pass