50 conf gen untils configuration file generation tool (#63)

* new: design config file v2.

* data_lifecycle: conf operation.

* rule: conf update.

* complex: conf update.

* core: split config.cc/h to DTCConfig&AutoConfig

* core: Merge cache/table_config_  into dtc_config

* conf-gen-utils: feature achievement.

* conf-gen-utils: shardingsphere conf add prefix.

* conf-gen-utils: fix bugs.

* core: alter conf.

* dtc: remove table.yaml, using dtc.yaml only.

* Connecotr: fixing diff table name at sing/mul tb.

* dtc: Changing relative path in logs.

* dtc: Changing relative path in logs.

* Conf-gen-utils: print a few logs.

* Conf-gen-utils: enhancement.

* core: Merge shm.id to listener.port

* libcommon: remove debug logs.

* DTC: Delete table.yaml, Using dtc.yaml only.

conf/dtc.yaml

@@ -1,18 +1,59 @@
-cache:
-   LOG_LEVEL: debug
-   BIND_ADDR: '*:20015/tcp'
-   DTCID: 20015
-   MAX_USE_MEM_MB: 100
-   DTC_MODE: 1 #1: cache only, 0: database in addition.
-vhot:
-   addr: 127.0.0.1:3307
-   username: root
-   password: root
-   database: sharding
+# 
+# DTC configure file. v2
+#
+props:
+  log.level: debug
+  listener.port.agent: 12001
+  listener.port.dtc: 20015
+  listener.port.extension: 3745
+  shm.mem.size: 100 #MB
+  #hash.custom: yes/no/enable/disable
+  #hash.custom.module: ../lib/key-hash.so
+  #hash.custom.functon: StringHash(1.128)
+
 data_lifecycle:
-   SingleQueryCount: 10
-   DataSQLRule: 'status = 0'
-   OperateTimeRule: '00 01 * * * ?'
-   LifeCycleTableName: 'data_lifecycle_table'
-   HotDBName: 'HOT'
-   ColdDBName: 'COLD'
+  single.query.count: 10
+  lifecycle.tablename: 'data_lifecycle_table'
+  rule.sql: 'status = 0'
+  rule.cron: '00 01 * * * ?'
+
+connection: &connection
+  addr: 127.0.0.1:3306
+  user: username
+  pwd: password 
+
+primary:
+  table: &table name
+  layered.rule: '(a>1 and a<3) or uid = 0'
+  cache:
+    field:
+      - {name: &key uid, type: signed/unsigned/float/string/binary, size: 4}
+      - {name: name, type: binary, size: 50}
+      - {name: city, type: string, size: 50}
+      - {name: sex, type: signed, size: 4}
+      - {name: age, type: signed, size: 4}
+  hot:
+    logic:
+      {db: &db L2, table: *table, connection: *connection}
+    real:
+      - {addr: mysql-01.local:3306, user: username, pwd: password, db: {prefix: &prefix [*db, _], start: 0, last: 10}}
+      - {addr: mysql-02.local:3306, user: username, pwd: password, db: {prefix: *prefix, start: 11, last: 25}}
+      - {addr: mysql-03.local:3306, user: username, pwd: password, db: {prefix: *prefix, start: 26, last: 29}}
+    sharding:
+      key: *key
+      table: {prefix: [*table, _], start: 0, last: 9}
+  full:
+    logic:
+      {db: L3, table: *table, connection: *connection}
+    real:
+      - {addr: 127.0.0.1:3306, user: username, pwd: password, db: L3}
+
+extension:
+  - logic:
+      {db: January, table: Monday, connection: *connection}
+    real:
+      - {addr: mysql-01.local:3306, user: username, pwd: password, db: January}
+  - logic:
+      {db: February, table: Tuesday, connection: *connection}
+    real:
+      - {addr: mysql-01.local:3306, user: username, pwd: password, db: February}

conf/log4cplus.conf

@@ -7,7 +7,7 @@ log4cplus.appender.STDOUT.layout.ConversionPattern=[%T] %D{%Y-%m-%d %H:%M:%S} %-
 
 ## synchronous log properties.
 log4cplus.appender.apdPlatform=log4cplus::TimeBasedRollingFileAppender
-log4cplus.appender.apdPlatform.FilenamePattern=/var/log/dtc/%d{yyyy-MM-dd}.log
+log4cplus.appender.apdPlatform.FilenamePattern=/var/log/dtc/dtc.%d{yyyyMMdd}.log
 log4cplus.appender.apdPlatform.Append=true
 log4cplus.appender.apdPlatform.MaxHistory=999
 log4cplus.appender.apdPlatform.ImmediateFlush=false

conf/table.yaml

@@ -1,68 +0,0 @@
-DATABASE_CONF:
-  hot_database_name: dtc_opensource 
-  hot_database_number: (1,1)        
-  hot_database_max_count: 1         
-  hot_server_count: 1               
-  hot_deploy: 0                      
-  cold_database_name: dtc_opensource
-  cold_database_number: (1,1)
-  cold_database_max_count: 1 
-  cold_server_count: 1
-  cold_deploy: 0
-  enable_key_hash: 0
-  key_hash_module: ../lib/key-hash.so
-  key_hash_function: StringHash(1,128)
-  
-HOT_MACHINE1:
-  Procs: 1
-  WriteProcs: 1
-  CommitProcs: 1
-  database_index: 0
-  database_address: 127.0.0.1:3306
-  database_username: username
-  database_password: password
-  MyCnf: ../conf/my.conf
-
-HOT_TABLE_CONF:
-  table_name: dtc_opensource
-  field_count: 5
-  key_count: 1
-  TableNum: (1,1)
-
-COLD_MACHINE1:
-  database_index: 0            
-  database_address: 127.0.0.1:3306
-  database_username: username
-  database_password: password
-  MyCnf: ../conf/my.conf
-  
-COLD_TABLE_CONF:
-  table_name: dtc_opensource            
-  field_count: 5                   
-  key_count: 1                     
-  TableNum: (1,1)          
-
-FIELD1:
-  field_name: uid
-  field_type: 1
-  field_size: 4
-
-FIELD2:
-  field_name: name
-  field_type: 5
-  field_size: 50
-        
-FIELD3:
-  field_name: city
-  field_type: 4
-  field_size: 50
-
-FIELD4:
-  field_name: sex
-  field_type: 1
-  field_size: 4
-  
-FIELD5:
-  field_name: age
-  field_type: 1
-  field_size: 4

src/CMakeLists.txt

@@ -3,6 +3,7 @@ ADD_SUBDIRECTORY(./libs)
 ADD_SUBDIRECTORY(./daemons)
 ADD_SUBDIRECTORY(./agent-watchdog)
 ADD_SUBDIRECTORY(./rule)
+ADD_SUBDIRECTORY(./utils)
 ADD_SUBDIRECTORY(./agent)
 ADD_SUBDIRECTORY(./core)
 ADD_SUBDIRECTORY(./connector)

src/agent-watchdog/CMakeLists.txt

@@ -12,6 +12,7 @@ LINK_DIRECTORIES(
     ${PROJECT_SOURCE_DIR}/build/src/libs/common
     ${PROJECT_SOURCE_DIR}/build/src/libs/stat)
 
+include(../utils.cmake)
 
 FILE(GLOB_RECURSE SRC_LIST1 ./*.cc ./*.c)
 
@@ -24,7 +25,9 @@ LINK_LIBRARIES(pthread)
 LINK_LIBRARIES(dl)
 
 ADD_DEFINITIONS("-g -fPIC -fpermissive -std=gnu++11")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_EXECUTABLE(agent-watchdog ${SRC_LIST1})
 
 TARGET_LINK_LIBRARIES(agent-watchdog libstat.a libcommon.a libyaml-cpp.a liblog4cplus.a)
+redefine_file_macro(agent-watchdog)

src/agent/CMakeLists.txt

@@ -15,6 +15,7 @@ IF("${HAVE_ASSERT_LOG}" EQUAL 1)
 ENDIF()
 
 FILE(GLOB_RECURSE  SRC_LIST ./*.c)
+include(../utils.cmake)
 
 INCLUDE_DIRECTORIES(
     .
@@ -37,7 +38,9 @@ LINK_LIBRARIES(pthread)
 LINK_LIBRARIES(librule.so)
 
 ADD_DEFINITIONS ("-D_GNU_SOURCE")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_EXECUTABLE(dtcagent ${SRC_LIST})
 
 TARGET_LINK_LIBRARIES(dtcagent libmxml.a librule.so)
+redefine_file_macro(agent-watchdog)

src/agent/da.c

@@ -343,7 +343,7 @@ static void show_usage(void) {
 			"  -D, --describe-stats   : print stats description and exit");
 	write_stderr(
 			"  -v, --verbosity=N      		: set logging level (default: 3, min: 1, max: 7)" CRLF
-			"  -o, --output=S         		: set logging dir (default: ../log/)" CRLF
+			"  -o, --output=S         		: set logging dir (default: /var/log/dtc/)" CRLF
 			"  -c, --conf-file=S      		: set configuration file (default: /etc/dtc/agent.xml)" CRLF
 			"  -e, --event-max-timeout=S	: set epoll max timeout(ms)(default: (30*1000)ms)" CRLF
 			"  -i, --stats_interval=S	    : set stats aggregator interval(ms)(default: (10*1000)ms)" CRLF
@@ -372,7 +372,7 @@ static bool da_test_conf(struct instance *dai) {
 }
 
 static void da_log_init(struct instance *dai) {
-	_init_log_("da", dai->log_dir);
+	_init_log_("agent", dai->log_dir);
 	_set_log_level_(dai->log_level);
 	_set_remote_log_fd_();
 }

src/agent/da_log.c

@@ -168,7 +168,7 @@ void _write_log_(int level, char *filename, const char *funcname,
 			int fd;
 
 			logday = today;
-			snprintf(logfile, sizeof(logfile), "%s/%s.error%04d%02d%02d.log",
+			snprintf(logfile, sizeof(logfile), "%s/%s.%04d%02d%02d.log",
 					log_dir, appname, tm.tm_year + 1900, tm.tm_mon + 1,
 					tm.tm_mday);
 			fd = open(logfile, O_CREAT | O_LARGEFILE | O_APPEND | O_WRONLY,

src/agent/da_stats.h

@@ -29,7 +29,7 @@ struct cache_instance;
 typedef void (*event_stats_cb_t)(void *, int);
 
 #define STATS_FILE "da.stats"
-#define STATS_DIR "../stats/"
+#define STATS_DIR "/var/log/dtc/stats/"
 #define STATS_INTERVAL (10 * 1000) /* in msec */
 #define DEFAULT_HTTP_TIMEOUT                                                   \
   5 /*default heep connect and request timeout in second*/

src/complex/CMakeLists.txt

@@ -1,5 +1,7 @@
 FILE(GLOB_RECURSE SRC_LIST ./*.cc ./*.c)
 
+include(../utils.cmake)
+
 #添加头文件搜索路径，相当于gcc -I
 INCLUDE_DIRECTORIES(
     .
@@ -27,9 +29,11 @@ LINK_LIBRARIES(dl)
 
 #编译参数
 ADD_DEFINITIONS("-g -rdynamic -fPIC -D_CORE_ -fpermissive -std=gnu++11 -D_GLIBCXX_USE_CXX11_ABI=0 -export-dynamic -Wl,--version-script,dtcd.export.lst")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 #编译为.so/.a
 ADD_EXECUTABLE(async-connector ${SRC_LIST})
 
 #将目标文件与库文件链接
-TARGET_LINK_LIBRARIES(async-connector libsqlparser.a liblog4cplus.a libyaml-cpp.a libnet.a libmysqlclient.a)
+TARGET_LINK_LIBRARIES(async-connector libsqlparser.a liblog4cplus.a libyaml-cpp.a libnet.a libmysqlclient.a)
+redefine_file_macro(async-connector)

src/complex/cm_load.cc

@@ -19,7 +19,6 @@
 #include "cm_conn.h"
 #include "log.h"
 
-#define TABLE_CONF_NAME "/etc/dtc/table.yaml"
 #define CACHE_CONF_NAME "/etc/dtc/dtc.yaml"
 
 using namespace std;
@@ -38,7 +37,6 @@ using namespace std;
 std::map<string, TableInfo> g_table_set;
 
 
-/* 前置空格已经过滤了 */
 static char* skip_blank(char *p)
 {
 	char *iter = p;
@@ -78,21 +76,6 @@ bool ConfigHelper::load_dtc_config()
 		log4cplus_error("load dtc file error:%s", e.what());
 		return false;
 	}
-
-	try 
-	{
-        table = YAML::LoadFile(TABLE_CONF_NAME);
-		if(table.IsNull())
-		{
-			log4cplus_error("table null");
-			return false;
-		}
-	}
-	catch (const YAML::Exception &e) 
-	{
-		log4cplus_error("load table file error:%s", e.what());
-		return false;
-	}
 	
 	if(!load_full_inst_info())
 	{
@@ -113,16 +96,13 @@ bool ConfigHelper::load_hot_inst_info()
 {
 	memset(&hot_instance, 0, sizeof(DBHost));
 
-	if(!dtc["vhot"])
-		return false;
-
-	if( !dtc["vhot"]["addr"] || 
-		!dtc["vhot"]["username"] ||
-		!dtc["vhot"]["password"] ||
-		!dtc["vhot"]["database"])
+	if( !dtc["primary"]["hot"]["logic"]["connection"]["addr"] || 
+		!dtc["primary"]["hot"]["logic"]["connection"]["user"] ||
+		!dtc["primary"]["hot"]["logic"]["connection"]["pwd"] ||
+		!dtc["primary"]["hot"]["logic"]["db"])
 		return false;
 
-	const char* addr = dtc["vhot"]["addr"].as<string>().c_str();
+	const char* addr = dtc["primary"]["hot"]["logic"]["connection"]["addr"].as<string>().c_str();
 	const char* p = strrchr(addr, ':');
 	if(p == NULL){
 		strncpy(hot_instance.Host, addr, sizeof(hot_instance.Host)-1 );
@@ -133,9 +113,9 @@ bool ConfigHelper::load_hot_inst_info()
 		hot_instance.Port = atoi(p+1);
 	}
 	
-	strncpy(hot_instance.User, dtc["vhot"]["username"].as<string>().c_str(), sizeof(hot_instance.User)-1 );
-	strncpy(hot_instance.Password, dtc["vhot"]["password"].as<string>().c_str(), sizeof(hot_instance.Password)-1 );
-	strncpy(hot_instance.DbName, dtc["vhot"]["database"].as<string>().c_str(), sizeof(hot_instance.DbName)-1 );
+	strncpy(hot_instance.User, dtc["primary"]["hot"]["logic"]["connection"]["user"].as<string>().c_str(), sizeof(hot_instance.User)-1 );
+	strncpy(hot_instance.Password, dtc["primary"]["hot"]["logic"]["connection"]["pwd"].as<string>().c_str(), sizeof(hot_instance.Password)-1 );
+	strncpy(hot_instance.DbName, dtc["primary"]["hot"]["logic"]["db"].as<string>().c_str(), sizeof(hot_instance.DbName)-1 );
 	
 	hot_instance.ConnTimeout = 10;
 	hot_instance.ReadTimeout = 10;
@@ -149,16 +129,16 @@ bool ConfigHelper::load_full_inst_info()
 {
 	memset(&full_instance, 0, sizeof(DBHost));
 
-	if(!table["COLD_MACHINE1"] || !table["DATABASE_CONF"])
+	if(!dtc["COLD_MACHINE1"] || !dtc["DATABASE_CONF"])
 		return false;
 
-	if( !table["COLD_MACHINE1"]["database_address"] || 
-		!table["COLD_MACHINE1"]["database_username"] ||
-		!table["COLD_MACHINE1"]["database_password"] ||
-		!table["DATABASE_CONF"]["cold_database_name"])
+	if( !dtc["primary"]["full"]["logic"]["connection"]["addr"] || 
+		!dtc["primary"]["full"]["logic"]["connection"]["user"] ||
+		!dtc["primary"]["full"]["logic"]["connection"]["pwd"] ||
+		!dtc["primary"]["full"]["logic"]["db"])
 		return false;
 
-	const char* addr = table["COLD_MACHINE1"]["database_address"].as<string>().c_str();
+	const char* addr = dtc["primary"]["full"]["logic"]["connection"]["addr"].as<string>().c_str();
 	const char* p = strrchr(addr, ':');
 	if(p == NULL){
 		strncpy(full_instance.Host, addr, sizeof(full_instance.Host)-1 );
@@ -169,9 +149,9 @@ bool ConfigHelper::load_full_inst_info()
 		full_instance.Port = atoi(p+1);
 	}
 
-	strncpy(full_instance.User, table["COLD_MACHINE1"]["database_username"].as<string>().c_str(), sizeof(full_instance.User)-1 );
-	strncpy(full_instance.Password, table["COLD_MACHINE1"]["database_password"].as<string>().c_str(), sizeof(full_instance.Password)-1 );
-	strncpy(full_instance.DbName, table["DATABASE_CONF"]["cold_database_name"].as<string>().c_str(), sizeof(full_instance.DbName)-1 );
+	strncpy(full_instance.User, dtc["primary"]["full"]["logic"]["connection"]["user"].as<string>().c_str(), sizeof(full_instance.User)-1 );
+	strncpy(full_instance.Password, dtc["primary"]["full"]["logic"]["connection"]["pwd"].as<string>().c_str(), sizeof(full_instance.Password)-1 );
+	strncpy(full_instance.DbName, dtc["primary"]["full"]["logic"]["db"].as<string>().c_str(), sizeof(full_instance.DbName)-1 );
 
 	full_instance.ConnTimeout = 10;
 	full_instance.ReadTimeout = 10;

src/complex/cm_load.h

@@ -13,7 +13,6 @@ class ConfigHelper
 {
 private:
 	std::string m_data;
-	YAML::Node table;
 	YAML::Node dtc;
 
 public:

src/connector/CMakeLists.txt

@@ -2,6 +2,8 @@
 
 FILE(GLOB SRC_LIST ./*.cc)
 
+include(../utils.cmake)
+
 INCLUDE_DIRECTORIES (
     ./
     ../core
@@ -43,7 +45,9 @@ LINK_LIBRARIES(libstat.a)
 LINK_LIBRARIES(libmysqlclient.a)
 
 ADD_DEFINITIONS ("-g -fPIC -fpermissive -std=gnu++11 -DOMN_PLATFORM_UNIX -Wl,--no-undefined -Xlinker -zmuldefs")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_EXECUTABLE (connector ${SRC_LIST})
 
-TARGET_LINK_LIBRARIES(connector  libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a  libyaml-cpp.a  libz64.a libmysqlclient.a)
+TARGET_LINK_LIBRARIES(connector  libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a  libyaml-cpp.a  libz64.a libmysqlclient.a)
+redefine_file_macro(connector)

src/connector/connector.cc

@@ -336,14 +336,15 @@ int main(int argc, char **argv)
 	if (!strcmp(addr, "-"))
 		fd = 0;
 	else {
-		if (strcasecmp(g_dtc_config->get_str_val("cache", "DTCID") ?:
-				       "",
-			       "none") != 0) {
+		std::string dtcid = g_dtc_config->get_config_node()["props"]["listener.port.dtc"].as<std::string>();
+		if(dtcid != "none")
+		{
 			log4cplus_warning(
 				"standalone %s need DTCID set to NONE",
 				progname);
 			return -1;
 		}
+		
 		SocketAddress sockaddr;
 		const char *err =
 			sockaddr.set_address(addr, argc == 2 ? NULL : argv[2]);

src/connector/mysql_operation.cc

@@ -1459,21 +1459,8 @@ void ConnectorProcess::set_title(const char *status)
 
 int ConnectorProcess::process_reload_config(DtcJob *Task)
 {
-    const char *keyStr = g_dtc_config->get_str_val("cache", "DTCID");
-    int cacheKey = 0;
-    if (keyStr == NULL) {
-        cacheKey = 0;
-        log4cplus_info("DTCID not set!");
-        return -1;
-    } else if (!strcasecmp(keyStr, "none")) {
-        log4cplus_warning("DTCID set to NONE, Cache disabled");
-        return -1;
-    } else if (isdigit(keyStr[0])) {
-        cacheKey = strtol(keyStr, NULL, 0);
-    } else {
-        log4cplus_warning("Invalid DTCID value \"%s\"", keyStr);
-        return -1;
-    }
+    int cacheKey = DbConfig::get_shm_id(g_dtc_config->get_config_node());;
+
     BlockProperties stInfo;
     BufferPond cachePool;
     memset(&stInfo, 0, sizeof(stInfo));

src/core/CMakeLists.txt

@@ -2,6 +2,8 @@ ADD_SUBDIRECTORY (./lib)
 
 FILE(GLOB_RECURSE SRC_LIST ./*.cc ./*.c)
 
+include(../utils.cmake)
+
 #添加头文件搜索路径，相当于gcc -I
 INCLUDE_DIRECTORIES(
     . ./algorithm ./buffer ./chain
@@ -43,9 +45,11 @@ LINK_LIBRARIES(dl)
 
 #编译参数
 ADD_DEFINITIONS("-g -rdynamic -fPIC -D_CORE_ -fpermissive -std=gnu++11 -D_GLIBCXX_USE_CXX11_ABI=0 -export-dynamic -Wl,--version-script,dtcd.export.lst")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 #编译为.so/.a
 ADD_EXECUTABLE(dtcd ${SRC_LIST})
 
 #将目标文件与库文件链接
 TARGET_LINK_LIBRARIES(dtcd libdaemons.a libstat.a libsqlparser.a libcommon.a libyaml-cpp.a liblog4cplus.a libz64.a libmysqlclient.a)
+redefine_file_macro(dtcd)

src/core/chain/buffer_process_ask_chain.cc

@@ -3040,6 +3040,7 @@ BufferProcessAskChain::buffer_column_expand_done(DTCJobOperation &Job)
 		char *bufLocal = (char *)MALLOC(strlen(buf) + 1);
 		memset(bufLocal, 0, strlen(buf) + 1);
 		strcpy(bufLocal, buf);
+
 		DbConfig *dbconfig = DbConfig::load_buffered(bufLocal);
 		FREE(bufLocal);
 		if (!dbconfig) {

src/core/data/col_expand.cc

@@ -71,6 +71,7 @@ int DTCColExpand::initialization()
 
 int DTCColExpand::reload_table()
 {
+	log4cplus_debug("reload table entry");
 	if (TableDefinitionManager::instance()->get_cur_table_idx() ==
 	    col_expand_->curTable)
 		return DTC_CODE_SUCCESS;
@@ -87,6 +88,8 @@ int DTCColExpand::reload_table()
 	}
 	TableDefinitionManager::instance()->set_cur_table_def(
 		t, col_expand_->curTable);
+
+	log4cplus_debug("reload table leave");
 	return DTC_CODE_SUCCESS;
 }
 

src/core/data/container_dtcd.cc

@@ -172,6 +172,7 @@ _QueryInternalService(const char *name, const char *instance)
 void init_task_executor(const char *name, AgentListenPool *listener,
 			JobAskInterface<DTCJobOperation> *main_chain)
 {
+	log4cplus_debug("name: %s, pool: %p, job: %p", name, listener, main_chain);
 	if (NCResultInternal::verify_class() == 0) {
 		log4cplus_error(
 			"Inconsistent class NCResultInternal detected, internal API disabled");

src/core/hotbk/hot_backup_ask_chain.cc

@@ -81,7 +81,7 @@ void HotBackupAskChain::job_ask_procedure(DTCJobOperation *job_operation)
 bool HotBackupAskChain::do_init(uint64_t total, off_t max_size)
 {
 	log4cplus_debug("total: %lu, max_size: %ld", total, max_size);
-	if (hbLog_.init("../log/hblog", "hblog", total, max_size)) {
+	if (hbLog_.init("/var/log/dtc/hblog", "hblog", total, max_size)) {
 		log4cplus_error("hotback process for hblog init failed");
 		return false;
 	}

src/core/main.cc

@@ -89,7 +89,7 @@ int main(int argc, char *argv[])
 	if (load_entry_parameter(argc, argv) < 0)
 		return DTC_CODE_FAILED;
 
-	if (g_dtc_config->get_int_val("cache", "EnableCoreDump", 0))
+	if (g_dtc_config->get_int_val("cache", "EnableCoreDump", 1))
 		init_core_dump();
 
 	if (init_config_info())

src/core/misc/main_supply.cc

@@ -87,8 +87,8 @@ int stat_open_fd()
 
 int init_cache_mode()
 {
-	g_datasource_mode = g_dtc_config->get_int_val("cache", "DTC_MODE",
-						      DTC_MODE_CACHE_ONLY);
+	DbConfig dbConfig;
+	g_datasource_mode = dbConfig.get_dtc_mode(g_dtc_config->get_config_node());
 	switch (g_datasource_mode) {
 	case DTC_MODE_DATABASE_ADDITION:
 		log4cplus_info("dtc datasource mode: %s(%d)",
@@ -114,20 +114,8 @@ int init_cache_mode()
 		return DTC_CODE_FAILED;
 	}
 
-	const char *keyStr = g_dtc_config->get_str_val("cache", "DTCID");
-	if (keyStr == NULL) {
-		cache_key = 0;
-	} else if (!strcasecmp(keyStr, "none") &&
-		   g_datasource_mode != DTC_MODE_DATABASE_ONLY) {
-		log4cplus_error(
-			"Can not set DTC_MODE=(DTC_MODE_CACHE_ONLY|DTC_MODE_DATABASE_ADDITION) and DTCID=NONE together.");
-		return DTC_CODE_FAILED;
-	} else if (isdigit(keyStr[0])) {
-		cache_key = strtol(keyStr, NULL, 0);
-	} else {
-		log4cplus_error("Invalid DTCID value \"%s\"", keyStr);
-		return DTC_CODE_FAILED;
-	}
+	cache_key = DbConfig::get_shm_id(g_dtc_config->get_config_node());
+	log4cplus_info("cache key:%d", cache_key);
 
 	if (g_datasource_mode == DTC_MODE_DATABASE_ONLY && async_update) {
 		log4cplus_error("can't DelayUpdate when DTCID set to NONE");
@@ -203,8 +191,8 @@ int init_buffer_process_ask_chain_thread()
 		return DTC_CODE_FAILED;
 	}
 
-	unsigned long long cache_size =
-		g_dtc_config->get_size_val("cache", "MAX_USE_MEM_MB", 0, 'M');
+	std::string str_size = DbConfig::get_shm_size(g_dtc_config->get_config_node());
+	unsigned long long cache_size = g_dtc_config->conv_size_val(str_size.c_str(), 0, 'M');
 	if (cache_size <= (50ULL << 20)) // 50M
 	{
 		log4cplus_error("MAX_USE_MEM_MB too small");
@@ -437,8 +425,8 @@ int init_buffer_process_ask_chain(PollerBase *thread)
 	g_buffer_process_ask_instance->set_limit_empty_nodes(
 		g_dtc_config->get_int_val("cache", "LimitEmptyNodes", 0));
 
-	unsigned long long cache_size =
-		g_dtc_config->get_size_val("cache", "MAX_USE_MEM_MB", 0, 'M');
+	std::string str_size = DbConfig::get_shm_size(g_dtc_config->get_config_node());
+	unsigned long long cache_size = g_dtc_config->conv_size_val(str_size.c_str(), 0, 'M');
 	if (cache_size <= (50ULL << 20)) // 50M
 	{
 		log4cplus_error("MAX_USE_MEM_MB too small");
@@ -593,9 +581,6 @@ int init_data_connector_ask_chain(PollerBase *thread)
 //获取、配置基础信息
 int init_config_info()
 {
-	// mkdir("/usr/local/dtc/stat", 0777);
-	// mkdir("/usr/local/dtc/data", 0777);
-
 	g_hash_changing = g_dtc_config->get_int_val("cache", "HashChanging", 0);
 	g_target_new_hash =
 		g_dtc_config->get_int_val("cache", "TargetNewHash", 0);

src/daemons/CMakeLists.txt

@@ -15,6 +15,7 @@ LINK_DIRECTORIES(
 
 FILE(GLOB_RECURSE SRC_LIST1 ./*.cc ./*.c)
 
+include(../utils.cmake)
 
 LINK_LIBRARIES(liblog4cplus.a) 
 LINK_LIBRARIES(libcommon.a)
@@ -24,9 +25,12 @@ LINK_LIBRARIES(pthread)
 LINK_LIBRARIES(dl)
 
 ADD_DEFINITIONS("-g -fPIC -fpermissive -std=gnu++11")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_EXECUTABLE(faultlogger ${SRC_LIST1})
 ADD_LIBRARY(daemons ${SRC_LIST1})
 
 TARGET_LINK_LIBRARIES(faultlogger libstat.a libcommon.a libyaml-cpp.a liblog4cplus.a)
 TARGET_LINK_LIBRARIES(daemons libstat.a liblog4cplus.a libyaml-cpp.a libcommon.a)
+redefine_file_macro(faultlogger)
+redefine_file_macro(daemons)

src/daemons/helper.cc

@@ -77,7 +77,7 @@ void WatchDogHelper::exec()
 	if (conf_ == DBHELPER_TABLE_NEW) {
 		argv[argc++] = (char *)"-t";// 4
 		char tableName[64];
-		snprintf(tableName, 64, "../conf/table%d.yaml", num_);
+		snprintf(tableName, 64, "/etc/dtc/dtc%d.yaml", num_);
 		argv[argc++] = tableName; // 5
 	} else if (conf_ == DBHELPER_TABLE_ORIGIN &&
 		   strcmp(daemons_table_file, TABLE_CONF_NAME)) {

src/daemons/hwc.cc

@@ -14,12 +14,12 @@ WatchDogHWC::~WatchDogHWC(void)
 void WatchDogHWC::exec(void)
 {
 	char* argv[6];
-	// ./hwcserver -d /etc/dtc/dtc.yaml -t /etc/dtc/table.yaml
+	// ./hwcserver -d /etc/dtc/dtc.yaml -t /etc/dtc/dtc.yaml
 	argv[0] = watchdog_object_name_;
 	argv[1] = (char*)"-d";
     argv[2] = "/etc/dtc/dtc.yaml";
 	argv[3] = (char*)"-t";
-	argv[4] = "/etc/dtc/table.yaml";
+	argv[4] = "/etc/dtc/dtc.yaml";
 	argv[5] = NULL;
 	execv(argv[0], argv);
 }

src/daemons/run.cc

@@ -26,6 +26,7 @@
 #include "config/dbconfig.h"
 #include "log/log.h"
 #include "daemon/daemon.h"
+#include "../core/global.h"
 /* 打开看门狗 */
 int start_dtc(int (*entry)(void *), void *args)
 {
@@ -34,7 +35,7 @@ int start_dtc(int (*entry)(void *), void *args)
     WatchDog *wdog = NULL;
     WatchDogListener *srv = NULL;
 
-	  if (g_dtc_config->get_int_val("cache", "DisableWatchDog", 0) == 0) {
+	  if (g_dtc_config->get_int_val("cache", "DisableWatchDog", 1) == 0) {
 		signal(SIGCHLD, SIG_DFL);
 		delay = g_dtc_config->get_int_val("cache", "WatchDogTime", 30);
 		if (delay < 5)
@@ -53,7 +54,8 @@ int start_dtc(int (*entry)(void *), void *args)
 			log4cplus_info("fork stat reporter");
 		}
 	}
-	if (g_dtc_config->get_int_val("cache", "DTC_MODE", 0) == 0) {
+	
+	if (DbConfig::get_dtc_mode(g_dtc_config->get_config_node()) == DTC_MODE_DATABASE_ADDITION) {
 		int nh = 0;
 		/* starting master helper */
 		for (int g = 0; g < dbConfig->machineCnt; ++g) {
@@ -104,7 +106,8 @@ int start_dtc(int (*entry)(void *), void *args)
 			return -1;
     // wait for dtc entry step to agentlisten
     usleep(100 * 1000);
-    if (g_dtc_config->get_int_val("cache", "EnableHwc", 1) > 0) {
+	
+    if (g_dtc_config->get_config_node()["primary"]["hot"]) {	// open hwcserver at DATABASE_IN_ADDITION mode.
             WatchDogHWC* p_hwc_wd = new WatchDogHWC(wdog, delay);
             if (p_hwc_wd->new_proc_fork() < 0) {
                 log4cplus_error("fork hwc server fail");

src/data_lifecycle/CMakeLists.txt

@@ -2,6 +2,8 @@ AUX_SOURCE_DIRECTORY(. SRC_LIST)
 AUX_SOURCE_DIRECTORY(. main)
 list(REMOVE_ITEM SRC_LIST "./cron_test.cc")
 
+include(../utils.cmake)
+
 INCLUDE_DIRECTORIES (
     .
     ../daemons
@@ -56,8 +58,10 @@ LINK_LIBRARIES(libstat.a)
 LINK_LIBRARIES(libmysqlclient.a)
 
 ADD_DEFINITIONS ("-g -fPIC -fpermissive -std=gnu++11 -DOMN_PLATFORM_UNIX -Wl,--no-undefined -Xlinker -zmuldefs")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 ADD_EXECUTABLE (data_lifecycle_manager ${SRC_LIST} ../connector/database_connection.cc)
 TARGET_LINK_LIBRARIES(data_lifecycle_manager libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a libyaml-cpp.a libz64.a libsqlparser.a libmysqlclient.a libmxml.a)
+redefine_file_macro(data_lifecycle_manager)
 
 if(jdtestOpen)
     cmake_minimum_required(VERSION 2.8.8)
@@ -108,6 +112,7 @@ if(jdtestOpen)
     ../libs/google_test/include
     )
     target_link_libraries(gtest_data_lifecycle dtcd daemons stat common gtest_main gmock gtest dl pthread log4cplus sqlparser yaml-cpp z64 mysqlclient mxml)
+    redefine_file_macro(gtest_data_lifecycle)
     SET_TARGET_PROPERTIES(gtest_data_lifecycle PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./bin")
     install(TARGETS gtest_data_lifecycle RUNTIME DESTINATION bin)
 endif()

src/data_lifecycle/README.md

@@ -18,11 +18,11 @@ dtc.yaml文件：
 
 ```
 data_lifecycle:
-   SingleQueryCount: 10 // 单次查询的记录条数
-   DataSQLRule: 'status = 0' // 定义数据规则的mysql语句
-   OperateTimeRule: '00 01 * * * ?' // 定义处理时机的时间规则，采用croncpp的格式，见https://github.com/mariusbancila/croncpp
-   LifeCycleDBName: 'data_lifecycle_database' // data_lifecycle表对应的库名，该表记录上次操作的数据对应的id、update_time等信息
-   LifeCycleTableName: 'data_lifecycle_table' // data_lifecycle表对应的表名
+   single.query.count: 10 // 单次查询的记录条数
+   rule.sql: 'status = 0' // 定义数据规则的mysql语句
+   rule.cron: '00 01 * * * ?' // 定义处理时机的时间规则，采用croncpp的格式，见https://github.com/mariusbancila/croncpp
+   lifecycle.dbname: 'data_lifecycle_database' // data_lifecycle表对应的库名，该表记录上次操作的数据对应的id、update_time等信息
+   lifecycle.tablename: 'data_lifecycle_table' // data_lifecycle表对应的表名
 ```
 
 table.yaml文件

src/data_lifecycle/data_conf.cc

@@ -7,7 +7,6 @@
 #include "daemon.h"
 #include "dbconfig.h"
 
-extern DTCConfig *g_dtc_config;
 extern DbConfig *dbConfig;
 extern char cache_file[256];
 char agent_file[256] = "/etc/dtc/agent.xml";
@@ -55,7 +54,7 @@ uint32_t DataConf::Port(){
 
 int DataConf::LoadConfig(int argc, char *argv[]){
     int c;
-    strcpy(table_file, "/etc/dtc/table.yaml");
+    strcpy(table_file, "/etc/dtc/dtc.yaml");
     strcpy(cache_file, "/etc/dtc/dtc.yaml");
 
     while ((c = getopt(argc, argv, "df:t:hvV")) != -1) {
@@ -81,12 +80,19 @@ int DataConf::LoadConfig(int argc, char *argv[]){
         }
     }
 
-    g_dtc_config = new DTCConfig;
-    if (0 != g_dtc_config->parse_config(cache_file, "data_lifecycle", false)){
+    YAML::Node config;
+    try {
+        config = YAML::LoadFile(cache_file);
+	} catch (const YAML::Exception &e) {
+		log4cplus_error("config file error:%s, %s\n", e.what(), cache_file);
+		return DTC_CODE_LOAD_CONFIG_ERR;
+	}
+
+    if(!config["data_lifecycle"]) {
         log4cplus_error("parse_config error.");
         return DTC_CODE_LOAD_CONFIG_ERR;
     }
-    if (0 != g_dtc_config->parse_config(table_file, "DATABASE_CONF", false)){
+    if(!config["primary"]) {
         log4cplus_error("parse_config error.");
         return DTC_CODE_LOAD_CONFIG_ERR;
     }
@@ -99,58 +105,53 @@ int DataConf::LoadConfig(int argc, char *argv[]){
 }
 
 int DataConf::ParseConfig(ConfigParam& config_param){
-    config_param.single_query_cnt_ = g_dtc_config->get_int_val("data_lifecycle", "SingleQueryCount", 10);
-    const char* data_rule = g_dtc_config->get_str_val("data_lifecycle", "DataSQLRule");
-    if(NULL == data_rule){
-        log4cplus_error("data_rule not defined.");
+    YAML::Node config;
+    try {
+        config = YAML::LoadFile(cache_file);
+	} catch (const YAML::Exception &e) {
+		log4cplus_error("config file error:%s, %s\n", e.what(), cache_file);
+		return DTC_CODE_LOAD_CONFIG_ERR;
+	}
+
+    YAML::Node node = config["data_lifecycle"]["single.query.count"];
+    config_param.single_query_cnt_ = node? node.as<int>(): 10;    
+    
+    node = config["data_lifecycle"]["rule.sql"];
+    if(!node){
+        log4cplus_error("rule.sql not defined.");
         return DTC_CODE_PARSE_CONFIG_ERR;
     }
-    config_param.data_rule_ = data_rule;
+    config_param.data_rule_ = node.as<string>();
 
-    const char* operate_time_rule = g_dtc_config->get_str_val("data_lifecycle", "OperateTimeRule");
-    if(NULL == operate_time_rule){
-        operate_time_rule = "00 01 * * * ?";
-    }
-    config_param.operate_time_rule_ = operate_time_rule;
+    node = config["data_lifecycle"]["rule.cron"];
+    config_param.operate_time_rule_ = node? node.as<string>(): "00 01 * * * ?";
 
     // 规则对应的操作operate_type  delete或update
-    const char* operate_type = g_dtc_config->get_str_val("data_lifecycle", "OperateType");
-    if(NULL == operate_type){
-        operate_type = "delete";
-    }
-    config_param.operate_type_ = operate_type;
+    node = config["data_lifecycle"]["type.operate"];
+    config_param.operate_type_ = node? node.as<string>(): "delete";
 
-    const char* life_cycle_table_name = g_dtc_config->get_str_val("data_lifecycle", "LifeCycleTableName");
-    if(NULL == life_cycle_table_name){
-        life_cycle_table_name = "data_lifecycle_table";
-    }
-    config_param.life_cycle_table_name_ = life_cycle_table_name;
+    node = config["data_lifecycle"]["lifecycle.tablename"];
+    config_param.life_cycle_table_name_ = node? node.as<string>(): "data_lifecycle_table";
 
-     const char* hot_db_name = g_dtc_config->get_str_val("data_lifecycle", "HotDBName");
-    if(NULL == hot_db_name){
-        hot_db_name = "L2";
-    }
-    config_param.hot_db_name_ = hot_db_name;
+    node = config["primary"]["hot"]["logic"]["db"];
+    config_param.hot_db_name_ = node? node.as<string>(): "L2";
 
-     const char* cold_db_name = g_dtc_config->get_str_val("data_lifecycle", "ColdDBName");
-    if(NULL == cold_db_name){
-        cold_db_name = "L3";
-    }
-    config_param.cold_db_name_ = cold_db_name;
+    node = config["primary"]["full"]["logic"]["db"];
+    config_param.cold_db_name_ = node? node.as<string>(): "L3";
 
-    const char* key_field_name = g_dtc_config->get_str_val("FIELD1", "field_name");
-    if(NULL == key_field_name){
+    node = config["primary"]["cache"]["field"][0]["name"];
+    if(!node){
         log4cplus_error("key_field_name not defined.");
         return DTC_CODE_PARSE_CONFIG_ERR;
     }
-    config_param.key_field_name_ = key_field_name;
+    config_param.key_field_name_ = node.as<string>();
 
-    const char* table_name = g_dtc_config->get_str_val("HOT_TABLE_CONF", "table_name");
-    if(NULL == table_name){
+    node = config["primary"]["hot"]["logic"]["table"];
+    if(!node){
         log4cplus_error("table_name not defined.");
         return DTC_CODE_PARSE_CONFIG_ERR;
     }
-    config_param.table_name_ = table_name;
+    config_param.table_name_ = node.as<string>();
 
     log4cplus_debug("single_query_cnt_: %d, data_rule: %s, operate_time_rule: %s, operate_type: %s, "
         "life_cycle_table_name: %s, key_field_name: %s, table_name: %s, hot_database_name: %s",

src/data_lifecycle/main.cc

@@ -35,9 +35,7 @@ int main(int argc, char *argv[]){
     /*if(NULL != p_data_manager){
         delete p_data_manager;
     }
-    if(NULL != g_dtc_config){
-        delete g_dtc_config;
-    }*/
+    */
     log4cplus_info("%s v%s: stopped", data_project_name, version);
     //Logger::shutdown();
     //daemon_cleanup();

src/hwcserver/CMakeLists.txt

@@ -1,6 +1,8 @@
 FILE(GLOB_RECURSE SRC_LIST ./*.cc  ../connector/*tion.cc)
 message(${SRC_LIST})
 
+include(../utils.cmake)
+
 INCLUDE_DIRECTORIES (
     ./
     ../core
@@ -46,7 +48,9 @@ LINK_LIBRARIES(libmysqlclient.a)
 LINK_LIBRARIES(libdtcapi.so)
 
 ADD_DEFINITIONS ("-g -fPIC -fpermissive -std=gnu++11 -DOMN_PLATFORM_UNIX -Wl,--no-undefined -Xlinker -zmuldefs")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_EXECUTABLE(hwcserver  ${SRC_LIST})
 
 TARGET_LINK_LIBRARIES(hwcserver  libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a  libyaml-cpp.a  libz64.a libmysqlclient.a libdtcapi.so)
+redefine_file_macro(hwcserver)

src/hwcserver/comm.cc

@@ -18,8 +18,8 @@ CRegistor CComm::registor;
 ConnectorProcess CComm::mysql_process_;
 
 const char *CComm::version = "hwc.0.1";
-char* CComm::dtc_conf = "../conf/dtc.yaml";
-char* CComm::table_conf = "../conf/table.yaml";
+char* CComm::dtc_conf = "/etc/dtc/dtc.yaml";
+char* CComm::table_conf = "/etc/dtc/dtc.yaml";
 int CComm::backend = 0;
 int CComm::normal = 1;
 
@@ -119,9 +119,8 @@ int CComm::connect_ttc_server(
     DbConfig* pParser)
 {
     log4cplus_warning("try to ping master server");
-
-    char* p_bind_addr = pParser->cfgObj->get_str_val("cache" , "BIND_ADDR");
-    if (NULL == p_bind_addr) {
+    char* p_bind_addr = pParser->get_bind_addr(pParser->cfgObj->get_config_node()).c_str();
+    if (strlen(p_bind_addr)) {
         return -1;
     }
     log4cplus_debug("master:%s.", p_bind_addr);

src/hwcserver/hwc_global.h

@@ -28,7 +28,7 @@
 
 #define MAX_ASYNC_FILE_SIZE         (10<<20)	//10M
 #define ASYNC_FILE_CONTROLLER_SIZE  512	//512bytes
-#define ASYNC_FILE_PATH             "../log/hwc"
+#define ASYNC_FILE_PATH             "/var/log/dtc/hwc"
 #define ASYNC_FILE_CONTROLLER       ASYNC_FILE_PATH"/controller"
 #define ASYNC_FILE_NAME             ASYNC_FILE_PATH"/hwc_"
 #define ASYNC_FILE_END_FLAG         0xFFFFFFFFUL
@@ -37,7 +37,7 @@
 #define READER_SLEEP_TIME           500	//500ms
 #define READER_RETRY_COUNT          20
 
-#define SYS_CONFIG_FILE             "../conf/hbp.conf"
+#define SYS_CONFIG_FILE             "/etc/dtc/hbp.conf"
 /*
  *   err code
  */

src/hwcserver/hwc_init_state.cc

@@ -47,16 +47,14 @@ void InitState::HandleEvent()
     // 解析yaml配置文件
     log4cplus_debug("dtc conf file:%s , table conf file:%s" , CComm::dtc_conf ,CComm::table_conf);
 	DTCConfig* p_dtc_config = new DTCConfig();
-	if (p_dtc_config->parse_config(CComm::table_conf, "DATABASE_CONF", false) == -1)
+	if (p_dtc_config->load_yaml_file(CComm::table_conf,  false) == -1)
 		return -1;
 
-	if (p_dtc_config->parse_config(CComm::dtc_conf, "cache", false))
+	if (p_dtc_config->load_yaml_file(CComm::dtc_conf, false))
 		return -1;
-
 	DbConfig* p_db_Config = DbConfig::Load(p_dtc_config , 1);
 	if (p_db_Config == NULL)
 		return -1;
-
     DTCTableDefinition* p_dtc_tab_def = p_db_Config->build_table_definition();
 
     TableDefinitionManager::instance()->set_cur_table_def(p_dtc_tab_def , 0);

src/libs/common/CMakeLists.txt

@@ -1,5 +1,7 @@
 FILE(GLOB_RECURSE SRC_LIST ./*.cc ./*.c)
 
+include(../../utils.cmake)
+
 INCLUDE_DIRECTORIES (
     .
     ../mysql/include
@@ -28,7 +30,9 @@ LINK_LIBRARIES(pthread)
 LINK_LIBRARIES(libmysqlclient.a)
 
 ADD_DEFINITIONS ("-g -fPIC -fpermissive -std=gnu++11")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_LIBRARY (common ${SRC_LIST})
 
-TARGET_LINK_LIBRARIES(common liblog4cplus.a libyaml-cpp.a libsqlparser.a libz64.a libmysqlclient.a)
+TARGET_LINK_LIBRARIES(common liblog4cplus.a libyaml-cpp.a libsqlparser.a libz64.a libmysqlclient.a)
+redefine_file_macro(common)

src/libs/common/agent/agent_listen_pool.cc

@@ -21,6 +21,8 @@
 #include "task/task_request.h"
 #include "log/log.h"
 #include "stat_dtc.h"
+#include "../config/dbconfig.h"
+#include "yaml-cpp/yaml.h"
 
 AgentListenPool::AgentListenPool()
 {
@@ -48,7 +50,7 @@ int AgentListenPool::register_entrance_chain_multi_thread(
 	DTCConfig *gc, JobAskInterface<DTCJobOperation> *next_chain)
 {
 	char bindstr[64];
-	const char *bindaddr;
+	std::string bindaddr;
 	const char *errmsg = NULL;
 	char thread_name[64];
 	int checktime;
@@ -63,12 +65,12 @@ int AgentListenPool::register_entrance_chain_multi_thread(
 		else
 			snprintf(bindstr, sizeof(bindstr), "BIND_ADDR%d", i);
 
-		bindaddr = gc->get_str_val("cache", bindstr);
-		if (NULL == bindaddr)
+		bindaddr = DbConfig::get_bind_addr(gc->get_config_node());
+		if (bindaddr.length() == 0)
 			continue;
 
 		if ((errmsg = socket_address[i].set_address(
-			     bindaddr, (const char *)NULL)))
+			     bindaddr.c_str(), (const char *)NULL)))
 			continue;
 
 		snprintf(thread_name, sizeof(thread_name), "dtc-thread-main-%d",
@@ -120,8 +122,7 @@ int AgentListenPool::register_entrance_chain(
 	DTCConfig *gc, JobAskInterface<DTCJobOperation> *next_chain,
 	PollerBase *bind_thread)
 {
-	char bindstr[64];
-	const char *bindaddr;
+	std::string bindaddr;
 	const char *errmsg = NULL;
 	int checktime;
 	int blog;
@@ -129,15 +130,13 @@ int AgentListenPool::register_entrance_chain(
 	checktime = gc->get_int_val("cache", "AgentRcvBufCheck", 5);
 	blog = gc->get_int_val("cache", "AgentListenBlog", 256);
 
-	snprintf(bindstr, sizeof(bindstr), "BIND_ADDR");
-
-	bindaddr = gc->get_str_val("cache", bindstr);
-	if (bindaddr == NULL) {
+	bindaddr = DbConfig::get_bind_addr(gc->get_config_node());
+	if (bindaddr == "") {
 		log4cplus_error("get cache BIND_ADDR configure failed");
 		return -1;
 	}
 
-	if ((errmsg = socket_address[0].set_address(bindaddr,
+	if ((errmsg = socket_address[0].set_address(bindaddr.c_str(),
 						    (const char *)NULL))) {
 		log4cplus_error("socket_address[0] setaddress failed");
 		return -1;

src/libs/common/config/auto_config.cc

@@ -0,0 +1,179 @@
+/*
+* Copyright [2021] JD.com, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <ctype.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <pthread.h>
+#include <sys/mman.h>
+
+#include "mem_check.h"
+#include "auto_config.h"
+#include "log/log.h"
+#include "config.h"
+
+class AutoConfigSection : public AutoConfig {
+    private:
+    static pthread_mutex_t glock;
+    void GlobalLock(void)
+    {
+        pthread_mutex_lock(&glock);
+    }
+    void GlobalUnlock(void)
+    {
+        pthread_mutex_unlock(&glock);
+    }
+
+    private:
+    DTCConfig *parent;
+    char *section;
+    // buf must have enough room place composed key name
+    char buf[256];
+    char *last;
+
+    public:
+    AutoConfigSection(DTCConfig *p, const char *sec);
+    ~AutoConfigSection();
+
+    virtual int get_int_val(const char *key, const char *inst, int def = 0);
+    virtual unsigned long long get_size_val(const char *key,
+                        const char *inst,
+                        unsigned long long def = 0,
+                        char unit = 0);
+    virtual int get_idx_val(const char *, const char *, const char *const *,
+                int = 0);
+    virtual const char *get_str_val(const char *key, const char *inst);
+
+    private:
+    // return composed key, or vanilla key, always non-null
+    const char *findkey(const char *key, const char *inst);
+    // strip suffix digits
+    int stripnumber(void);
+    // strip suffix alphaphetic
+    int stripalpha(void);
+    // strip suffix punct
+    int strippunct(void);
+};
+
+pthread_mutex_t AutoConfigSection::glock = PTHREAD_MUTEX_INITIALIZER;
+
+AutoConfigSection::AutoConfigSection(DTCConfig *p, const char *sec)
+{
+    this->parent = p;
+    this->section = STRDUP(sec);
+}
+
+AutoConfigSection::~AutoConfigSection()
+{
+    FREE_CLEAR(section);
+}
+
+int AutoConfigSection::stripnumber(void)
+{
+    int n = 0;
+    while (last >= buf && isdigit(*last)) {
+        last--;
+        n++;
+    }
+    last[1] = 0;
+    strippunct();
+    return n;
+}
+
+int AutoConfigSection::stripalpha(void)
+{
+    int n = 0;
+    while (last >= buf && isalpha(*last)) {
+        last--;
+        n++;
+    }
+    last[1] = 0;
+    strippunct();
+    return n;
+}
+
+int AutoConfigSection::strippunct(void)
+{
+    int n = 0;
+    while (last >= buf && *last != '@' && !isalnum(*last)) {
+        last--;
+        n++;
+    }
+    last[1] = 0;
+    return n;
+}
+
+const char *AutoConfigSection::findkey(const char *key, const char *inst)
+{
+    snprintf(buf, sizeof(buf), "%s@%s", key, inst);
+    last = buf + strlen(buf) - 1;
+    strippunct();
+
+    do {
+        if (parent->get_str_val(section, buf) != NULL) {
+            return buf;
+        }
+    } while (isdigit(*last) ? stripnumber() : stripalpha());
+
+    return key;
+}
+
+int AutoConfigSection::get_int_val(const char *key, const char *inst, int def)
+{
+    int ret;
+    GlobalLock();
+    ret = parent->get_int_val(section, findkey(key, inst), def);
+    GlobalUnlock();
+    return ret;
+}
+unsigned long long AutoConfigSection::get_size_val(const char *key,
+                           const char *inst,
+                           unsigned long long def,
+                           char unit)
+{
+    unsigned long long ret;
+    GlobalLock();
+    ret = parent->get_size_val(section, findkey(key, inst), def, unit);
+    GlobalUnlock();
+    return ret;
+}
+int AutoConfigSection::get_idx_val(const char *key, const char *inst,
+                   const char *const *idxval, int def)
+{
+    int ret;
+    GlobalLock();
+    ret = parent->get_idx_val(section, findkey(key, inst), idxval, def);
+    GlobalUnlock();
+    return ret;
+}
+const char *AutoConfigSection::get_str_val(const char *key, const char *inst)
+{
+    const char *ret;
+    GlobalLock();
+    ret = parent->get_str_val(section, findkey(key, inst));
+    GlobalUnlock();
+    return ret;
+}
+
+AutoConfig *DTCConfig::get_auto_config_instance(const char *section)
+{
+    AutoConfigSection *inst;
+    NEW(AutoConfigSection(this, section), inst);
+    return inst;
+}

src/libs/common/config/auto_config.h

@@ -0,0 +1,40 @@
+/*
+* Copyright [2021] JD.com, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#ifndef __AUTO_CONFIG_H__
+#define __AUTO_CONFIG_H__
+
+#include <stdio.h>
+#include <string.h>
+#include <string>
+#include <map>
+#include "yaml-cpp/yaml.h"
+
+class AutoConfig {
+    public:
+	AutoConfig(){};
+	virtual ~AutoConfig(){};
+	virtual int get_int_val(const char *key, const char *inst,
+				int def = 0) = 0;
+	virtual unsigned long long get_size_val(const char *key,
+						const char *inst,
+						unsigned long long def = 0,
+						char unit = 0) = 0;
+	virtual int get_idx_val(const char *, const char *, const char *const *,
+				int = 0) = 0;
+	virtual const char *get_str_val(const char *key, const char *inst) = 0;
+};
+
+#endif

src/libs/common/config/config.cc

@@ -62,9 +62,8 @@ int DTCConfig::Dump(const char *fn, bool dec)
     FILE *fp = fopen(fn, "w");
     if (fp == NULL)
         return -1;
-    //fprintf(fp, "##### ORIGINAL FILE %s #####\n", filename_.c_str());
-    for (YAML::const_iterator ite = table_config_.begin();
-         ite != table_config_.end(); ++ite) {
+    for (YAML::const_iterator ite = dtc_config.begin();
+         ite != dtc_config.end(); ++ite) {
         YAML::Node inception_sec = ite->first;
         std::string sec = inception_sec.as<std::string>();
         fprintf(fp, "%s:\n", sec.c_str());
@@ -86,43 +85,34 @@ int DTCConfig::Dump(const char *fn, bool dec)
     return 0;
 }
 
-int DTCConfig::parse_buffered_config(char *buf, const char *fn,
-                     const char *defsec, bool bakconfig)
+int DTCConfig::load_yaml_buffer(char *buf)
 {
     int ret_code = -1;
 
     if(!buf)
     {
-        log4cplus_debug("buffer null.");
+        log4cplus_error("yaml content don't allow null.");
         return ret_code;
     }
-
+    log4cplus_debug("yaml buf:%s", buf);
     try {
-        if (defsec && (strcmp(defsec, "cache") == 0 || strcmp(defsec, "data_lifecycle") == 0)) {
-            cache_config_ = YAML::Load(buf);
-        } else {
-            table_config_ = YAML::Load(buf);
-        }
+        dtc_config = YAML::Load(buf);
         ret_code = 0;
     } catch (const YAML::Exception &e) {
-        printf("config file error:%s\n", e.what());
-        log4cplus_debug("config file error:%s\n", e.what());
+        printf("load yaml buf error:%s\n", e.what());
+        log4cplus_debug("load yaml buf error:%s\n", e.what());
         return ret_code;
     }
     return ret_code;
 }
 
-int DTCConfig::parse_config(const char *fn, const char *defsec, bool bakconfig)
+int DTCConfig::load_yaml_file(const char *fn, bool bakconfig)
 {
     int ret_code = -1;
 
     try {
         printf("open config file:%s\n", fn);
-        if (defsec && (strcmp(defsec, "cache") == 0 || strcmp(defsec, "data_lifecycle") == 0)) {
-            cache_config_ = YAML::LoadFile(fn);
-        } else {
-            table_config_ = YAML::LoadFile(fn);
-        }
+        dtc_config = YAML::LoadFile(fn);
         ret_code = 0;
     } catch (const YAML::Exception &e) {
         printf("config file error:%s\n", e.what());
@@ -132,127 +122,33 @@ int DTCConfig::parse_config(const char *fn, const char *defsec, bool bakconfig)
     if (bakconfig) {
         char bak_config[1024];
         int err = 0;
-        system("mkdir -p /usr/local/dtc/stat/");
+        system("mkdir -p /etc/dtc/stat/");
         snprintf(bak_config, sizeof(bak_config),
-             "cp %s /usr/local/dtc/stat/", fn);
+             "cp %s /etc/dtc/stat/", fn);
         if (err == 0)
             err = system(bak_config);
     }
     return ret_code;
 }
 
-bool DTCConfig::has_section(const char *sec)
-{
-    YAML::Node inception = table_config_[sec];
-    if (inception && inception.IsMap())
-        return true;
-    else
-        return false;
-}
-
-bool DTCConfig::has_key(const char *sec, const char *key)
-{
-    if (!table_config_[sec])
-        return false;
-
-    YAML::Node inception = table_config_[sec][key];
-    if (inception && inception.IsScalar())
-        return true;
-    else
-        return false;
-}
-
-/********************************************
- * eg：get_str_val("cache", "RemoteLogAddr")
- * 读取 conf/cache.conf中RemoteLogAdd的参数
- * ******************************************/
 const char *DTCConfig::get_str_val(const char *sec, const char *key)
 {
-    if (sec && (strcmp(sec, "cache") == 0 || strcmp(sec, "data_lifecycle") == 0)) {
-        if (cache_config_[sec]) {
-            if (cache_config_[sec][key]) {
-                std::string s_cache = cache_config_[sec][key]
-                                 .as<std::string>();
-                if (s_cache.length() > 0) {
-                    return strdup(s_cache.c_str());
-                }
-            }
-        }
-    } else {
-        if (table_config_[sec]) {
-            if (table_config_[sec][key]) {
-                std::string s_cache = table_config_[sec][key]
-                                 .as<std::string>();
-                if (s_cache.length() > 0) {
-                    log4cplus_info("cache str:%s", s_cache.c_str());
-                    return strdup(s_cache.c_str());
-                }
-            }
-        }
-    }
-
     return NULL;
 }
 
 int DTCConfig::get_int_val(const char *sec, const char *key, int def)
 {
-    const char *val = NULL;
-    if (sec && (strcmp(sec, "cache") == 0 || strcmp(sec, "data_lifecycle") == 0)) {
-        if (cache_config_[sec]) {
-            if (cache_config_[sec][key]) {
-                std::string result = cache_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
-    } else {
-        if (table_config_[sec]) {
-            if (table_config_[sec][key]) {
-                std::string result = table_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
-    }
-
-    if (val == NULL)
-        return def;
-
-    return str2int(val, def);
+    return def;
 }
 
-unsigned long long DTCConfig::get_size_val(const char *sec, const char *key,
-                       unsigned long long def, char unit)
+unsigned long long DTCConfig::conv_size_val(const char* val, int ndefault, char unit)
 {
-    const char *val = NULL;
-    if (sec && (strcmp(sec, "cache") == 0 || strcmp(sec, "data_lifecycle") == 0)) {
-        if (cache_config_[sec]) {
-            if (cache_config_[sec][key]) {
-                std::string result = cache_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
-    } else {
-        if (table_config_[sec]) {
-            if (table_config_[sec][key]) {
-                std::string result = table_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
+    if (val == NULL)
+    {
+        log4cplus_error("val is null");
+        return ndefault;
     }
-
-    if (val == NULL || !isdigit(val[0]))
-        return def;
+        
 
     const char *p;
     double a = strtod(val, (char **)&p);
@@ -292,198 +188,14 @@ unsigned long long DTCConfig::get_size_val(const char *sec, const char *key,
     return (unsigned long long)a;
 }
 
-int DTCConfig::get_idx_val(const char *sec, const char *key,
-               const char *const *array, int nDefault)
-{
-    const char *val = NULL;
-    if (sec && (strcmp(sec, "cache") == 0 || strcmp(sec, "data_lifecycle") == 0)) {
-        if (cache_config_[sec]) {
-            if (cache_config_[sec][key]) {
-                std::string result = cache_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
-    } else {
-        if (table_config_[sec]) {
-            if (table_config_[sec][key]) {
-                std::string result = table_config_[sec][key]
-                                 .as<std::string>();
-                if (result.length() > 0) {
-                    val = result.c_str();
-                }
-            }
-        }
-    }
-    if (val == NULL)
-        return nDefault;
-
-    if (isdigit(val[0])) {
-        char *p;
-        int n = strtol(val, &p, 0);
-        if (*p == '\0') {
-            for (int i = 0; array[i]; ++i) {
-                if (n == i)
-                    return i;
-            }
-        }
-    }
-
-    for (int i = 0; array[i]; ++i) {
-        if (!strcasecmp(val, array[i]))
-            return i;
-    }
-    return -1;
-}
-
-class AutoConfigSection : public AutoConfig {
-    private:
-    static pthread_mutex_t glock;
-    void GlobalLock(void)
-    {
-        pthread_mutex_lock(&glock);
-    }
-    void GlobalUnlock(void)
-    {
-        pthread_mutex_unlock(&glock);
-    }
-
-    private:
-    DTCConfig *parent;
-    char *section;
-    // buf must have enough room place composed key name
-    char buf[256];
-    char *last;
-
-    public:
-    AutoConfigSection(DTCConfig *p, const char *sec);
-    ~AutoConfigSection();
-
-    virtual int get_int_val(const char *key, const char *inst, int def = 0);
-    virtual unsigned long long get_size_val(const char *key,
-                        const char *inst,
-                        unsigned long long def = 0,
-                        char unit = 0);
-    virtual int get_idx_val(const char *, const char *, const char *const *,
-                int = 0);
-    virtual const char *get_str_val(const char *key, const char *inst);
-
-    private:
-    // return composed key, or vanilla key, always non-null
-    const char *findkey(const char *key, const char *inst);
-    // strip suffix digits
-    int stripnumber(void);
-    // strip suffix alphaphetic
-    int stripalpha(void);
-    // strip suffix punct
-    int strippunct(void);
-};
-
-pthread_mutex_t AutoConfigSection::glock = PTHREAD_MUTEX_INITIALIZER;
-
-AutoConfigSection::AutoConfigSection(DTCConfig *p, const char *sec)
-{
-    this->parent = p;
-    this->section = STRDUP(sec);
-}
-
-AutoConfigSection::~AutoConfigSection()
-{
-    FREE_CLEAR(section);
-}
-
-int AutoConfigSection::stripnumber(void)
-{
-    int n = 0;
-    while (last >= buf && isdigit(*last)) {
-        last--;
-        n++;
-    }
-    last[1] = 0;
-    strippunct();
-    return n;
-}
-
-int AutoConfigSection::stripalpha(void)
-{
-    int n = 0;
-    while (last >= buf && isalpha(*last)) {
-        last--;
-        n++;
-    }
-    last[1] = 0;
-    strippunct();
-    return n;
-}
-
-int AutoConfigSection::strippunct(void)
-{
-    int n = 0;
-    while (last >= buf && *last != '@' && !isalnum(*last)) {
-        last--;
-        n++;
-    }
-    last[1] = 0;
-    return n;
-}
-
-const char *AutoConfigSection::findkey(const char *key, const char *inst)
-{
-    snprintf(buf, sizeof(buf), "%s@%s", key, inst);
-    last = buf + strlen(buf) - 1;
-    strippunct();
-
-    do {
-        if (parent->get_str_val(section, buf) != NULL) {
-            return buf;
-        }
-    } while (isdigit(*last) ? stripnumber() : stripalpha());
-
-    return key;
-}
-
-int AutoConfigSection::get_int_val(const char *key, const char *inst, int def)
-{
-    int ret;
-    GlobalLock();
-    ret = parent->get_int_val(section, findkey(key, inst), def);
-    GlobalUnlock();
-    return ret;
-}
-unsigned long long AutoConfigSection::get_size_val(const char *key,
-                           const char *inst,
-                           unsigned long long def,
-                           char unit)
-{
-    unsigned long long ret;
-    GlobalLock();
-    ret = parent->get_size_val(section, findkey(key, inst), def, unit);
-    GlobalUnlock();
-    return ret;
-}
-int AutoConfigSection::get_idx_val(const char *key, const char *inst,
-                   const char *const *idxval, int def)
-{
-    int ret;
-    GlobalLock();
-    ret = parent->get_idx_val(section, findkey(key, inst), idxval, def);
-    GlobalUnlock();
-    return ret;
-}
-const char *AutoConfigSection::get_str_val(const char *key, const char *inst)
+unsigned long long DTCConfig::get_size_val(const char *sec, const char *key,
+                       unsigned long long def, char unit)
 {
-    const char *ret;
-    GlobalLock();
-    ret = parent->get_str_val(section, findkey(key, inst));
-    GlobalUnlock();
-    return ret;
+    return def;
 }
 
-AutoConfig *DTCConfig::get_auto_config_instance(const char *section)
+int DTCConfig::get_idx_val(const char *sec, const char *key,
+               const char *const *array, int nDefault)
 {
-    AutoConfigSection *inst;
-    NEW(AutoConfigSection(this, section), inst);
-    return inst;
+    return nDefault;
 }

src/libs/common/config/config.h

@@ -20,22 +20,10 @@
 #include <string.h>
 #include <string>
 #include <map>
+#include "auto_config.h"
 #include "yaml-cpp/yaml.h"
 
-class AutoConfig {
-    public:
-	AutoConfig(){};
-	virtual ~AutoConfig(){};
-	virtual int get_int_val(const char *key, const char *inst,
-				int def = 0) = 0;
-	virtual unsigned long long get_size_val(const char *key,
-						const char *inst,
-						unsigned long long def = 0,
-						char unit = 0) = 0;
-	virtual int get_idx_val(const char *, const char *, const char *const *,
-				int = 0) = 0;
-	virtual const char *get_str_val(const char *key, const char *inst) = 0;
-};
+int str2int(const char *strval, int def);
 
 class DTCConfig {
     public:
@@ -50,19 +38,16 @@ class DTCConfig {
 	int get_idx_val(const char *, const char *, const char *const *,
 			int = 0);
 	const char *get_str_val(const char *sec, const char *key);
-	const YAML::Node& get_array_node(const char* sec , const char* key = NULL);
+	unsigned long long conv_size_val(const char* val, int ndefault, char unit);
 
-	bool has_section(const char *sec);
-	bool has_key(const char *sec, const char *key);
 	int Dump(const char *fn, bool dec = false);
-	int parse_config(const char *f = 0, const char *s = 0,
-			 bool bakconfig = false);
-	int parse_buffered_config(char *buf, const char *fn = 0,
-				  const char *s = 0, bool bakconfig = false);
+	int load_yaml_file(const char *f = 0, bool bakconfig = false);
+	int load_yaml_buffer(char *buf);
+
+	YAML::Node get_config_node() { return dtc_config;}
 
     private:
-	YAML::Node table_config_;
-	YAML::Node cache_config_;
+	YAML::Node dtc_config;
 };
 
 #endif

src/libs/common/config/dbconfig.h

@@ -90,7 +90,20 @@ struct KeyHash {
 	key_hash_interface keyHashFunction;
 };
 
-struct DbConfig {
+enum Layered {
+	HOT = 0,
+	FULL
+};
+
+enum Depoly{
+	SINGLE = 0,
+	SHARDING_DB_ONE_TAB = 1,
+	SINGLE_DB_SHARDING_TAB = 2,
+	SHARDING_DB_SHARDING_TAB = 3
+};
+
+class DbConfig {
+public:
 	DTCConfig *cfgObj;
 	char *dbName;
 	char *dbFormat;
@@ -109,7 +122,7 @@ struct DbConfig {
 	int machineCnt;
 	int procs; //all machine procs total
 	int database_max_count; //max db index
-	char depoly; //0:none 1:multiple db 2:multiple table 3:both
+	enum Depoly depoly;
 
 	struct KeyHash keyHashConfig;
 
@@ -139,9 +152,13 @@ struct DbConfig {
 	bool Compare(DbConfig *config, bool compareMach = false);
 	int find_new_mach(DbConfig *config, std::vector<int> &newMach,
 			  std::map<int, int> &machMap);
+	static int get_dtc_mode(YAML::Node dtc_config);
+	static std::string get_shm_size(YAML::Node dtc_config);
+	static int get_shm_id(YAML::Node dtc_config);
+	static std::string get_bind_addr(YAML::Node dtc_config);
 
 private:
-	int parse_db_config(DTCConfig* raw, int i_server_type = 0);
+	int get_dtc_config(YAML::Node dtc_config, DTCConfig* raw,int i_server_type);
 	int convert_case_sensitivity(std::string& s_val);
 };
 

src/libs/common/config/dbconfig_tdef.cc

@@ -20,6 +20,7 @@
 DTCTableDefinition *DbConfig::build_table_definition(void)
 {
 	DTCTableDefinition *tdef = new DTCTableDefinition(fieldCnt);
+	log4cplus_debug("build table tblName:%s", tblName);
 	tdef->set_table_name(tblName);
 	for (int i = 0; i < fieldCnt; i++) {
 		if (tdef->add_field(i, field[i].name, field[i].type,

src/libs/common/config/parse_cluster_config.h

@@ -21,7 +21,6 @@
 #include <map>
 namespace ClusterConfig
 {
-//#define CLUSTER_CONFIG_FILE "../conf/ClusterConfig.xml"
 #define CLUSTER_CONFIG_FILE "/etc/dtc/clusterconfig.xml"
 struct ClusterNode {
 	std::string name;

src/libs/common/daemon/daemon.cc

@@ -42,7 +42,7 @@ int background = 1;
 const char stat_project_name[] = "daemon";
 const char stat_usage_argv[] = "";
 
-#define TABLE_CONF_NAME "/etc/dtc/table.yaml"
+#define TABLE_CONF_NAME "/etc/dtc/dtc.yaml"
 #define CACHE_CONF_NAME "/etc/dtc/dtc.yaml"
 
 char d_cache_file[256] = CACHE_CONF_NAME;
@@ -106,20 +106,18 @@ int load_entry_parameter(int argc, char **argv)
 
 	//init_log("dtcd");
 	log4cplus_info("%s v%s: starting....", stat_project_name, version);
-	strcpy(d_table_file, "/etc/dtc/table.yaml");
+	strcpy(d_table_file, "/etc/dtc/dtc.yaml");
 	strcpy(d_cache_file, "/etc/dtc/dtc.yaml");
 	g_dtc_config = new DTCConfig;
 	//load config file and copy it to ../stat
-	if (g_dtc_config->parse_config(d_table_file, "DATABASE_CONF", true) == -1)
+	if (g_dtc_config->load_yaml_file(d_table_file, true) == -1)
 		return -1;
 
-	if (g_dtc_config->parse_config(d_cache_file, "cache", true))
+	if (g_dtc_config->load_yaml_file(d_cache_file, true))
 		return -1;
-
 	dbConfig = DbConfig::Load(g_dtc_config);
 	if (dbConfig == NULL)
 		return -1;
-
 	g_table_def[0] = dbConfig->build_table_definition();
 	if (g_table_def[0] == NULL)
 		return -1;
@@ -237,8 +235,8 @@ int init_core_dump(void)
 	struct rlimit rlim;
 
 	/* allow core dump  100M */
-	rlim.rlim_cur = 100UL << 20;
-	rlim.rlim_max = 100UL << 20;
+	rlim.rlim_cur = 100UL << 30;
+	rlim.rlim_max = 100UL << 30;
 	//设置core文件的最大字节数
 	if (setrlimit(RLIMIT_CORE, &rlim) == -1) {
 		//如果设置失败，将软限制设置为硬限制，并重新设置

src/libs/common/data_connector_ask_chain.cc

@@ -238,7 +238,12 @@ int DataConnectorAskChain::build_helper_object(int idx)
 	}
 
 	DTCConfig* p_dtc_conf = dbConfig[idx]->cfgObj;
-	int i_has_hwc = p_dtc_conf ? p_dtc_conf->get_int_val("cache", "EnableHwc", 1) : 1;
+	int i_has_hwc = 0;
+	if(p_dtc_conf)
+	{
+		if(p_dtc_conf->get_config_node()["primary"]["hot"])
+			i_has_hwc = 1;
+	}
 	log4cplus_info("enable hwc:%d" , i_has_hwc);
 
 	/* build helper object */
@@ -296,15 +301,19 @@ int DataConnectorAskChain::build_master_group_mapping(int idx)
 	for (int i = 0; i < dbConfig[idx]->database_max_count; i++)
 		groupMap[idx][i] = GMAP_NONE;
 
+	log4cplus_info("machine cnt:%d", dbConfig[idx]->machineCnt);
 	/* build master group mapping */
 	for (int i = 0; i < dbConfig[idx]->machineCnt; i++) {
 		int gm_id = i;
+		log4cplus_info("helper type:%d", dbConfig[idx]->mach[i].helperType);
 		if (dbConfig[idx]->mach[i].helperType == DUMMY_HELPER) {
 			gm_id = GMAP_DUMMY;
 			hasDummyMachine = 1;
 		} else if (dbConfig[idx]->mach[i].procs == 0) {
+			log4cplus_error("procs=0 at idx:%d, i: %d", idx, i);
 			continue;
 		}
+		log4cplus_info("mach[%d].dbCnt: %d", i, dbConfig[idx]->mach[i].dbCnt);
 		for (int j = 0; j < dbConfig[idx]->mach[i].dbCnt; j++) {
 			const int db = dbConfig[idx]->mach[i].dbIdx[j];
 			if (groupMap[idx][db] >= 0) {
@@ -317,6 +326,7 @@ int DataConnectorAskChain::build_master_group_mapping(int idx)
 		}
 	}
 	for (int i = 0; i < dbConfig[idx]->database_max_count; ++i) {
+		log4cplus_info("database_max_count:%d, idx: %d", dbConfig[idx]->database_max_count, idx);
 		if (groupMap[idx][i] == GMAP_NONE) {
 			log4cplus_error(
 				"db completeness check error, db %d not found",
@@ -344,7 +354,7 @@ DbConfig *DataConnectorAskChain::get_db_config(DTCJobOperation *job)
 			strlen(row[3].bin.ptr), row[3].bin.ptr);
 	char *buf = row[3].bin.ptr;
 	config = new DTCConfig();
-	if (config->parse_buffered_config(buf, NULL, "DATABASE_CONF", false) !=
+	if (config->load_yaml_buffer(buf) !=
 	    0) {
 		log4cplus_error(
 			"table.yaml illeagl when migrate db, parse error");
@@ -414,7 +424,7 @@ int DataConnectorAskChain::migrate_db(DTCJobOperation *job)
 
 	// save new table.yaml as table%d.conf
 	char tableName[64];
-	snprintf(tableName, 64, "../conf/table%d.conf", tableNo);
+	snprintf(tableName, 64, "/etc/dtc/dtc%d.conf", tableNo);
 	log4cplus_debug("table.yaml: %s", tableName);
 	if (dbConfig[1]->cfgObj->Dump(tableName, true) != 0) {
 		log4cplus_error("save table.yaml as table2.conf error");
@@ -487,7 +497,7 @@ int DataConnectorAskChain::switch_db(DTCJobOperation *job)
 	dbConfig[1]->destory();
 	dbConfig[1] = NULL;
 	// write conf file
-	dbConfig[0]->cfgObj->Dump("../conf/table.yaml", false);
+	dbConfig[0]->cfgObj->Dump("/etc/dtc/dtc.yaml", false);
 	Cleanup();
 
 	return 0;

src/libs/common/dtc_global.h

@@ -17,7 +17,7 @@
 #define _DTC_GLOBAL_H_
 #include "algorithm/non_copyable.h"
 
-#define TABLE_CONF_NAME "/etc/dtc/table.yaml"
+#define TABLE_CONF_NAME "/etc/dtc/dtc.yaml"
 #define CACHE_CONF_NAME "/etc/dtc/dtc.yaml"
 #define ALARM_CONF_FILE "/etc/dtc/dtcalarm.conf"
 

src/libs/common/key/key_route_ask_chain.cc

@@ -613,7 +613,7 @@ bool KeyRouteAskChain::migration_inprogress()
 	return false;
 }
 
-static const char *state_file_name = "../data/cluster.stat";
+static const char *state_file_name = "/var/log/dtc/data/cluster.stat";
 
 void KeyRouteAskChain::save_state_to_file()
 {

src/libs/common/key/key_route_ask_chain.h

@@ -78,7 +78,7 @@ class KeyRouteAskChain : public JobAskInterface<DTCJobOperation> {
 
 	std::string key_list_file_name(const std::string &name)
 	{
-		return "../data/" + name + ".migrated";
+		return "/var/log/dtc/data/" + name + ".migrated";
 	}
 
 	std::string select_node(const char *key);

src/libs/common/listener/listener_pool.cc

@@ -78,23 +78,22 @@ int ListenerPool::do_bind(DTCConfig *gc, JobAskInterface<DTCJobOperation> *out)
 	for (int i = 0; i < MAXLISTENERS; i++) {
 		const char *errmsg;
 		char bindStr[32];
-		char bindPort[32];
 		int rbufsz;
 		int wbufsz;
 
 		if (i == 0) {
-			snprintf(bindStr, sizeof(bindStr), "BIND_ADDR");
-			snprintf(bindPort, sizeof(bindPort), "BindPort");
+			snprintf(bindStr, sizeof(bindStr), "listener.port.dtc");
 		} else {
-			snprintf(bindStr, sizeof(bindStr), "BIND_ADDR%d", i);
-			snprintf(bindPort, sizeof(bindPort), "BindPort%d", i);
+			snprintf(bindStr, sizeof(bindStr), "listener.port.dtc.%d", i);
 		}
 
-		const char *addrStr = gc->get_str_val("cache", bindStr);
-		if (addrStr == NULL)
+		int port = gc->get_config_node()["props"][bindStr].as<int>();
+        char sz[200] = {0};
+        sprintf(sz, "*:%d/tcp", port);
+        std::string addrStr = sz;
+		if (addrStr.length() == 0)
 			continue;
-		errmsg = sockaddr[i].set_address(
-			addrStr, gc->get_str_val("cache", bindPort));
+		errmsg = sockaddr[i].set_address(addrStr.c_str(), (const char*)NULL);
 		if (errmsg) {
 			log4cplus_error("bad BIND_ADDR%d/BindPort%d: %s\n", i,
 					i, errmsg);

src/libs/common/packet/packet_server.cc

@@ -46,7 +46,7 @@ enum enum_select_types {
 const MetaSelections meta_selections[] = {
 	{"select dtctables" , E_SELECT_DTC_TABLES 	, NULL},
 	{"select dtcyaml" 	, E_SELECT_DTC_YAML 	, "/etc/dtc/dtc.yaml"},
-	{"select tableyaml" , E_SELECT_TABLE_YAML 	, "/etc/dtc/table.yaml"}
+	{"select tableyaml" , E_SELECT_TABLE_YAML 	, "/etc/dtc/dtc.yaml"}
 };
 
 enum enum_field_types { MYSQL_TYPE_DECIMAL, MYSQL_TYPE_TINY,
@@ -1317,6 +1317,7 @@ int Packet::load_table(const char* p_filename, char*& file , int& i_length)
 		return -1;
 	}
 
+	printf("open file:%s\n", p_filename);
 	lseek(fd, 0L, SEEK_SET);
 	i_length = lseek(fd, 0L, SEEK_END);
 	lseek(fd, 0L, SEEK_SET);

src/libs/common/table/table_def.cc

@@ -309,7 +309,6 @@ int DTCTableDefinition::set_key_fields(int n)
 		case DField::Unsigned: //整数
 		case DField::Float: //浮点数
 			maxSize += field_size(i);
-			log4cplus_info("cyj001:%d" , maxSize);
 			break;
 		case DField::String: //以null结尾的字符串
 		case DField::Binary: //二进制数据
@@ -329,7 +328,6 @@ int DTCTableDefinition::set_key_fields(int n)
 	if (uniqFieldCnt == n && keysAsUniqField > 0)
 		keysAsUniqField = 1; /* EXACT */
 	if (nvar == 0) { // not string , binary
-		log4cplus_info("cyj002:%d" , maxKeySize);
 		if (maxKeySize >= 256) 
 			return -1;
 		keyFormat = maxSize; // key 总大小

src/libs/common/table/table_def_manager.cc

@@ -68,7 +68,7 @@ bool TableDefinitionManager::renew_cur_table_def()
 
 bool TableDefinitionManager::save_new_table_conf()
 {
-	_save_dbconfig->cfgObj->Dump("../conf/table.yaml", false);
+	_save_dbconfig->cfgObj->Dump("/etc/dtc/dtc.yaml", false);
 	_save_dbconfig->destory();
 	_save_dbconfig = NULL;
 	return true;

src/libs/common/thread/thread.cc

@@ -114,10 +114,6 @@ void Thread::auto_config_stack_size(void)
 void Thread::auto_config_cpu_mask(void)
 {
 #ifdef CPU_ZERO
-	const char *val = g_autoconf->get_str_val("ThreadCPUMask", taskname);
-	if (val != NULL) {
-		cpumask = strtoll(val, NULL, 16);
-	}
 	log4cplus_debug("autoconf thread %s ThreadCPUMask %llx", taskname,
 			(unsigned long long)cpumask);
 #endif

src/libs/net/config.cc

@@ -228,7 +228,7 @@ int CConfig::ParseBufferedConfig(char *buf, const char *fn, const char *defsec,
 	{
 			/*add by foryzhou write config file to stat directory,ignore error*/
 			char bak_config[1024];
-			snprintf(bak_config,sizeof(bak_config),"cp %s ../stat/",fn);
+			snprintf(bak_config,sizeof(bak_config),"cp %s /etc/dtc/stat/",fn);
 			system(bak_config);
 	}
 	return ret_code;

src/libs/stat/CMakeLists.txt

@@ -9,6 +9,8 @@ LINK_DIRECTORIES(
     ${PROJECT_SOURCE_DIR}/src/libs/yaml-cpp/libs
     ${PROJECT_SOURCE_DIR}/build/src/libs/common)
 
+include(../../utils.cmake)
+
 FILE(GLOB_RECURSE SRC_LIST1 ./*.cc ./*.c)
 
 LINK_LIBRARIES(liblog4cplus.a) 
@@ -18,9 +20,10 @@ LINK_LIBRARIES(pthread)
 LINK_LIBRARIES(dl)
 
 ADD_DEFINITIONS("-g -fPIC -fpermissive -std=gnu++11")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 ADD_LIBRARY(stat ${SRC_LIST1})
 
 TARGET_LINK_LIBRARIES(stat liblog4cplus.a libyaml-cpp.a libcommon.a)
-
+redefine_file_macro(stat)
 

src/libs/stat/stat_manager.cc

@@ -113,7 +113,7 @@ int StatManager::init_stat_info(const char *name, const char *indexfile,
 	fd = open(indexfile, O_RDWR);
 	if (fd < 0) {
 		snprintf(stat_error_message_, sizeof(stat_error_message_),
-			 "cannot open index file");
+			 "cannot open index file, checking privilege and stat directory.");
 		return -1;
 	}
 

src/rule/CMakeLists.txt

@@ -1,5 +1,7 @@
 FILE(GLOB_RECURSE SRC_LIST ./*.cc ./*.c)
 
+include(../utils.cmake)
+
 #添加头文件搜索路径，相当于gcc -I
 INCLUDE_DIRECTORIES(
     .
@@ -22,10 +24,12 @@ LINK_LIBRARIES(dl)
 
 #编译参数
 ADD_DEFINITIONS("-g -rdynamic -fPIC -D_CORE_ -fpermissive -std=gnu++11 -D_GLIBCXX_USE_CXX11_ABI=0 -export-dynamic -Wl,--version-script,dtcd.export.lst")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
 
 #编译为.so/.a
 #ADD_EXECUTABLE(rule ${SRC_LIST})
 ADD_LIBRARY(rule SHARED ${SRC_LIST})
 
 #将目标文件与库文件链接
-TARGET_LINK_LIBRARIES(rule libsqlparser.so liblog4cplus.a libyaml-cpp.a)
+TARGET_LINK_LIBRARIES(rule libsqlparser.so liblog4cplus.a libyaml-cpp.a)
+redefine_file_macro(conf-gen-utils)

src/rule/re_load.cc

@@ -5,9 +5,7 @@
 #include <iostream>
 #include "re_comm.h"
 
-#define TABLE_CONF_NAME "/etc/dtc/table.yaml"
 #define CACHE_CONF_NAME "/etc/dtc/dtc.yaml"
-#define ALARM_CONF_FILE "/etc/dtc/dtcalarm.conf"
 
 using namespace hsql;
 hsql::SQLParserResult rule_ast;
@@ -25,13 +23,11 @@ std::string do_get_rule()
 		return "";
 	}
 
-    if(config["match"])
+    YAML::Node node = config["primary"]["layered.rule"];
+    if(node)
     {
-        if(config["match"]["RULE"])
-        {
-            std::string rules = config["match"]["RULE"].as<string>();
-            return rules;
-        }
+        std::string rules = node.as<string>();
+        return rules;
     }
 
     return "";
@@ -169,32 +165,21 @@ extern "C" int re_load_table_key(char* key)
 {
     YAML::Node config;
     try {
-        config = YAML::LoadFile(TABLE_CONF_NAME);
+        config = YAML::LoadFile(CACHE_CONF_NAME);
 	} catch (const YAML::Exception &e) {
 		log4cplus_error("config file error:%s\n", e.what());
 		return -1;
 	}
 
-    if(config["TABLE_CONF"])
+    YAML::Node node = config["primary"]["cache"]["field"][0]["name"];
+    if(node)
     {
-        if(config["TABLE_CONF"]["key_count"])
+        if(node.as<string>().length() >= 50)
         {
-            if(config["TABLE_CONF"]["key_count"].as<int>() == 1)
-            {
-                if(config["FIELD1"])
-                {
-                    if(config["FIELD1"]["field_name"])
-                    {
-                        if(config["FIELD1"]["field_name"].as<string>().length() >= 50)
-                        {
-                            return -1;
-                        }
-                        strcpy(key, config["FIELD1"]["field_name"].as<string>().c_str());
-                        return 0;
-                    }
-                }
-            }
+            return -1;
         }
+        strcpy(key, node.as<string>().c_str());
+        return 0;
     }
 
     return -1;

src/utils.cmake

@@ -0,0 +1,16 @@
+# utils.cmake
+
+function(redefine_file_macro targetname)
+    get_target_property(source_files "${targetname}" SOURCES)
+    foreach(sourcefile ${source_files})
+        get_property(defs SOURCE "${sourcefile}"
+            PROPERTY COMPILE_DEFINITIONS)
+        get_filename_component(filepath "${sourcefile}" ABSOLUTE)
+        string(REPLACE ${PROJECT_SOURCE_DIR}/ "" relpath ${filepath})
+        list(APPEND defs "__FILE__=\"${relpath}\"")
+        set_property(
+            SOURCE "${sourcefile}"
+            PROPERTY COMPILE_DEFINITIONS ${defs}
+            )
+    endforeach()
+endfunction()

src/utils/CMakeLists.txt

@@ -0,0 +1,34 @@
+FILE(GLOB_RECURSE SRC_LIST ./*.cc ./*.c)
+
+include(../utils.cmake)
+
+#添加头文件搜索路径，相当于gcc -I
+INCLUDE_DIRECTORIES(
+    .
+    ../libs/yaml-cpp/include/
+    ../libs/log4cplus/include/
+    ../libs/hsql/include)
+
+#添加.so/.a文件搜索路径，相当于gcc -L
+LINK_DIRECTORIES(
+    ${PROJECT_SOURCE_DIR}/src/libs/log4cplus/libs
+    ${PROJECT_SOURCE_DIR}/src/libs/yaml-cpp/libs
+    ${PROJECT_SOURCE_DIR}/src/libs/hsql/libs)
+
+#添加链接库，相当于gcc -l
+LINK_LIBRARIES(liblog4cplus.a) 
+LINK_LIBRARIES(libsqlparser.so) 
+LINK_LIBRARIES(libyaml-cpp.a)
+LINK_LIBRARIES(pthread)
+LINK_LIBRARIES(dl)
+
+#编译参数
+ADD_DEFINITIONS("-g -rdynamic -fPIC -D_CORE_ -fpermissive -std=gnu++11 -D_GLIBCXX_USE_CXX11_ABI=0 -export-dynamic -Wl,--version-script,dtcd.export.lst")
+ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
+
+#编译为.so/.a
+ADD_EXECUTABLE(conf-gen-utils ${SRC_LIST})
+
+#将目标文件与库文件链接
+TARGET_LINK_LIBRARIES(conf-gen-utils libsqlparser.so liblog4cplus.a libyaml-cpp.a)
+redefine_file_macro(conf-gen-utils)

src/utils/conf-gen-utils.cc

@@ -0,0 +1,349 @@
+#include <stdio.h>
+#include <iostream>
+#include "log.h"
+#include "yaml-cpp/yaml.h"
+#include <vector>
+#include <map>
+
+using namespace std;
+#define ROOT_PATH "/etc/dtc/"
+
+char conf_file[256] = {0};
+YAML::Node dtc_config;
+std::map<std::string, std::vector<YAML::Node>> dbmap;
+std::string get_merge_string(YAML::Node node);
+std::map<std::string, std::string> algorithm;
+
+int load_dtc_config(int argc, char *argv[])
+{
+    int c;
+    strcpy(conf_file, ROOT_PATH);
+    strcat(conf_file, "dtc.yaml");
+
+    while ((c = getopt(argc, argv, "c:")) != -1) {
+        switch (c) {
+        case 'c':
+            log4cplus_info("conf file:%s", optarg);
+            strncpy(conf_file, optarg, sizeof(conf_file) - 1);
+            break;
+        }
+    }
+
+    try {
+        log4cplus_info("loading file: %s", conf_file);
+        dtc_config = YAML::LoadFile(conf_file);
+	} catch (const YAML::Exception &e) {
+		log4cplus_error("config file error:%s, %s\n", e.what(), conf_file);
+		return -1;
+	}
+
+    if(!dtc_config)
+        return -1;
+
+    log4cplus_info("loading conf file successfully.");
+    return 0;
+}
+
+int load_node_to_map()
+{
+    log4cplus_info("loading node to map.");
+    std::vector<YAML::Node> vec;
+    //hot
+    if(dtc_config["primary"]["hot"])
+    {
+        log4cplus_info("loading hot.");
+        vec = 
+            dbmap[dtc_config["primary"]["hot"]["logic"]["db"].as<string>()];
+        vec.push_back(dtc_config["primary"]["hot"]);
+        dbmap[dtc_config["primary"]["hot"]["logic"]["db"].as<string>()] = vec;
+    }
+
+    //full
+    if(dtc_config["primary"]["full"])
+    {
+        log4cplus_info("loading full.");
+        vec = dbmap[dtc_config["primary"]["full"]["logic"]["db"].as<string>()];
+        vec.push_back(dtc_config["primary"]["full"]);
+        dbmap[dtc_config["primary"]["full"]["logic"]["db"].as<string>()] = vec;
+    }
+
+    //extension
+    if(dtc_config["extension"])
+    {
+        log4cplus_info("loading extension.");
+        for(int i = 0; i < dtc_config["extension"].size(); i++)
+        {
+            vec = dbmap[dtc_config["extension"][i]["logic"]["db"].as<string>()];
+            vec.push_back(dtc_config["extension"][i]);
+            dbmap[dtc_config["extension"][i]["logic"]["db"].as<string>()] = vec;
+        }
+    }
+
+    log4cplus_info("loading node to map finished.");
+    return 0;
+}
+
+int yaml_dump_schema_name(FILE *fp, std::string logic_db_name)
+{
+    fprintf(fp, "databaseName: %s\n", logic_db_name.c_str());
+    return 0;
+}
+
+int yaml_dump_data_sources(FILE *fp, std::vector<YAML::Node> vec)
+{
+    fprintf(fp, "dataSources:\n");
+    std::vector<YAML::Node>::iterator vt;
+    for(vt = vec.begin(); vt != vec.end(); vt++)
+    {
+        YAML::Node node = *vt;
+        for(int i = 0; i < node["real"].size(); i++)
+        {
+            if(node["real"][i]["db"].IsScalar())  //single db
+            {
+                fprintf(fp, "  %s:\n", node["real"][i]["db"].as<string>().c_str());
+                fprintf(fp, "    url: jdbc:mysql://%s/%s?serverTimezone=UTC&useSSL=false&zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=UTF-8\n",
+                    node["real"][i]["addr"].as<string>().c_str(), node["real"][i]["db"].as<string>().c_str());
+
+                fprintf(fp, "    username: %s\n", node["real"][i]["user"].as<string>().c_str());
+                fprintf(fp, "    password: %s\n", node["real"][i]["pwd"].as<string>().c_str());
+                fprintf(fp, "    connectionTimeoutMilliseconds: %d\n", 30000);
+                fprintf(fp, "    idleTimeoutMilliseconds: %d\n", 60000);
+                fprintf(fp, "    maxLifetimeMilliseconds: %d\n", 1800000);
+                fprintf(fp, "    maxPoolSize: %d\n", 50);
+                fprintf(fp, "    minPoolSize: %d\n", 1);
+            }
+            else    //multi db
+            {
+                for(int j = node["real"][i]["db"]["start"].as<int>(); j <= node["real"][i]["db"]["last"].as<int>(); j++)
+                {
+                    char szdb[250] = {0};
+                    sprintf(szdb, "%s%d", get_merge_string(node["real"][i]["db"]["prefix"]).c_str(), j);
+                    fprintf(fp, "  %s:\n", szdb);
+
+                    fprintf(fp, "    url: jdbc:mysql://%s/%s?serverTimezone=UTC&useSSL=false&zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=UTF-8\n",
+                        node["real"][i]["addr"].as<string>().c_str(), szdb);
+
+                    fprintf(fp, "    username: %s\n", node["real"][i]["user"].as<string>().c_str());
+                    fprintf(fp, "    password: %s\n", node["real"][i]["pwd"].as<string>().c_str());
+                    fprintf(fp, "    connectionTimeoutMilliseconds: %d\n", 30000);
+                    fprintf(fp, "    idleTimeoutMilliseconds: %d\n", 60000);
+                    fprintf(fp, "    maxLifetimeMilliseconds: %d\n", 1800000);
+                    fprintf(fp, "    maxPoolSize: %d\n", 50);
+                    fprintf(fp, "    minPoolSize: %d\n", 1);
+                }
+            }
+            
+            
+        }
+
+
+    }
+    return 0;
+}
+
+std::string get_merge_string(YAML::Node node)
+{
+    std::string str = "";
+    if(!node)
+        return str;
+
+    for(int i = 0; i < node.size(); i++)
+    {
+        str += node[i].as<string>();
+    }
+
+    return str;
+}
+
+std::string yaml_dump_actual_data_nodes(YAML::Node node)
+{
+    std::string str = "";
+    //single db
+    if(node["real"].size() == 1 && node["real"][0]["db"].IsScalar())    
+    {
+        str += node["real"][0]["db"].as<string>();
+    }
+    else    //multi db
+    {
+        str += get_merge_string(node["real"][0]["db"]["prefix"]);
+        char sztmp[200] = {0};
+        sprintf(sztmp, "${%d..%d}", 
+            node["real"][0]["db"]["start"].as<int>(),
+            node["real"][node["real"].size()-1]["db"]["last"].as<int>());
+        str += sztmp;
+    }
+
+    str += ".";
+
+    //single table
+    if(!node["sharding"])
+    {
+        str += node["logic"]["table"].as<string>();
+    }
+    else    //multi table
+    {
+        str += get_merge_string(node["sharding"]["table"]["prefix"]);
+        char sztmp[200] = {0};
+        sprintf(sztmp, "{%d..%d}", 
+            node["sharding"]["table"]["start"].as<int>(),
+            node["sharding"]["table"]["last"].as<int>());
+        str += sztmp;
+    }
+
+    return str;
+}
+
+int yaml_dump_sharding_rule(FILE *fp, std::vector<YAML::Node> vec)
+{
+    fprintf(fp, "rules:\n");
+    fprintf(fp, "- !SHARDING\n");
+    fprintf(fp, "  tables:\n");
+    std::string binding_table = "";
+    std::vector<YAML::Node>::iterator vt;
+    for(vt = vec.begin(); vt != vec.end(); vt++)
+    {
+        YAML::Node node = *vt;
+        int dbcount = 1;
+        std::string tbname = node["logic"]["table"].as<string>();
+        binding_table += tbname;
+        if(vt + 1 != vec.end())
+            binding_table += ",";
+        fprintf(fp, "    %s:\n", tbname.c_str());
+
+        fprintf(fp, "      actualDataNodes: %s\n", yaml_dump_actual_data_nodes(node).c_str());
+
+        if(node["sharding"])
+        {
+            std::string key = node["sharding"]["key"].as<string>();
+
+            if(node["real"].size() > 1 || node["real"][0]["db"].IsMap())
+            {
+                dbcount = node["real"][node["real"].size()-1]["db"]["last"].as<int>() - 
+                    node["real"][0]["db"]["start"].as<int>() + 1;
+                fprintf(fp, "      databaseStrategy:\n");
+                fprintf(fp, "        standard:\n");
+                fprintf(fp, "          shardingColumn: %s\n", key.c_str());
+                char szname[1024] = {0};
+                sprintf(szname, "%s_db_inline", tbname.c_str());
+                fprintf(fp, "          shardingAlgorithmName: %s\n", szname);
+
+                char sztemp[1024] = {0};
+                sprintf(sztemp, "$%s{(%s%%%d)}", 
+                    get_merge_string(node["real"][0]["db"]["prefix"]).c_str(),
+                    key.c_str(),
+                    dbcount);
+                algorithm[szname] = sztemp;
+            }
+            
+            fprintf(fp, "      tableStrategy:\n");
+            fprintf(fp, "        standard:\n");
+            fprintf(fp, "          shardingColumn: %s\n", key.c_str());
+            char szname[1024] = {0};
+            sprintf(szname, "%s_tb_inline", tbname.c_str());
+            fprintf(fp, "          shardingAlgorithmName: %s\n", szname);
+            char sztemp[1024] = {0};
+            sprintf(sztemp, "%s${((%s/%d).longValue()%%%d)}", 
+                get_merge_string(node["sharding"]["table"]["prefix"]).c_str(),
+                key.c_str(),
+                dbcount,
+                node["sharding"]["table"]["last"].as<int>() -
+                node["sharding"]["table"]["start"].as<int>() + 1);
+            algorithm[szname] = sztemp;
+
+            fprintf(fp, "      keyGenerateStrategy:\n");
+            fprintf(fp, "        column: %s\n", key.c_str());
+            fprintf(fp, "        keyGeneratorName: snowflake\n");
+        }
+    }
+
+    fprintf(fp, "  shardingAlgorithms:\n");
+    std::map<std::string, std::string>::iterator it;
+    for (it = algorithm.begin(); it != algorithm.end(); it++) 
+    {
+        fprintf(fp, "    %s:\n", (*it).first.c_str());
+        fprintf(fp, "      type: INLINE\n");
+        fprintf(fp, "      props:\n");
+        fprintf(fp, "        algorithm-expression: %s\n", (*it).second.c_str());
+    }
+
+    fprintf(fp, "  bindingTables:\n");
+    fprintf(fp, "    - %s\n", binding_table.c_str());
+
+    return 0;
+}
+
+
+
+int dump_single_conf_file(std::string logic_db_name, std::vector<YAML::Node> vec)
+{
+    std::string filename = string(ROOT_PATH) + string("config-") + logic_db_name + ".yaml";
+    FILE *fp = fopen(filename.c_str(), "w");
+    if (fp == NULL)
+        return -1;
+    yaml_dump_schema_name(fp, logic_db_name);
+    yaml_dump_data_sources(fp, vec);
+    yaml_dump_sharding_rule(fp, vec);
+    fclose(fp);
+    log4cplus_info("generating new conf file:%s", filename.c_str());
+    return 0;
+}
+
+int dump_shardingsphere_conf_files()
+{
+    log4cplus_info("dumping ss conf files.");
+    std::map<std::string, std::vector<YAML::Node>>::iterator it;
+    for (it = dbmap.begin(); it != dbmap.end(); it++) 
+    {
+        std::string logic_db_name = (*it).first;
+        std::vector<YAML::Node> vec = (*it).second;
+        dump_single_conf_file(logic_db_name, vec);        
+    }
+
+    log4cplus_info("dumping ss conf files finished.");
+    return 0;
+}
+
+void delete_all_old_yaml()
+{
+    std::string cmd = "rm -rf ";
+    cmd += ROOT_PATH;
+    cmd += "config-*.yaml";
+    log4cplus_info("cmd: %s", cmd.c_str());
+    system(cmd.c_str());
+}
+
+void dump_authority()
+{
+    std::string filename = string(ROOT_PATH) + "server.yaml";
+    FILE *fp = fopen(filename.c_str(), "w");
+    if (fp == NULL)
+        return;
+
+    fprintf(fp, "rules:\n");
+    fprintf(fp, "  - !AUTHORITY\n");
+    fprintf(fp, "    users:\n");
+    fprintf(fp, "      - sharding@%%:sharding\n");
+    fprintf(fp, "    provider:\n");
+    fprintf(fp, "      type: ALL_PERMITTED\n");
+
+    fclose(fp);
+}
+
+int main(int argc, char* argv[])
+{
+    init_log4cplus();
+    log4cplus_info("************* conf-gen-utils *************");
+
+    if(load_dtc_config(argc, argv) < 0)
+        return 0;
+
+    load_node_to_map();
+
+    delete_all_old_yaml();
+
+    dump_shardingsphere_conf_files();
+
+    dump_authority();
+
+    return 0;
+}

src/utils/log.cc

@@ -0,0 +1,72 @@
+/*
+* Copyright [2021] JD.com, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#include <stdlib.h>
+#include <fcntl.h>
+#include <time.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include "log.h"
+void init_log4cplus()
+{
+	PropertyConfigurator::doConfigure(LOG4CPLUS_TEXT(LOG4CPLUS_CONF_FILE));
+}
+
+void write_log(Logger logger, int level, const char *file_name,
+	       const char *func_name, int line, const char *fmt, ...)
+{
+	//eg:[test.cpp] - [main] : 28--
+	char str[4096];
+	sprintf(str, "%d", line);
+	string msg = "[" + (string)file_name + " : " + (string)str + "] - [" +
+		     (string)func_name + "]" + +" -- ";
+	char buf[4096];
+	int i;
+	va_list ap;
+	va_start(ap, fmt);
+	i = vsnprintf(buf, 4096, fmt, ap);
+	char *tmp = buf;
+	string s = msg + string(tmp);
+	va_end(ap);
+	switch (level) {
+	case 1:
+		LOG4CPLUS_TRACE(logger, s);
+		break;
+	case 2:
+		LOG4CPLUS_DEBUG(logger, s);
+		break;
+	case 3:
+		LOG4CPLUS_INFO(logger, s);
+		break;
+	case 4:
+		LOG4CPLUS_WARN(logger, s);
+		break;
+	case 5:
+		LOG4CPLUS_ERROR(logger, s);
+		break;
+	case 6:
+		LOG4CPLUS_FATAL(logger, s);
+		break;
+	default:
+		break;
+	}
+}

src/utils/log.h

@@ -0,0 +1,98 @@
+/*
+* Copyright [2021] JD.com, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#ifndef __DTCLOG_H__
+#define __DTCLOG_H__
+
+#include <sys/cdefs.h>
+__BEGIN_DECLS
+
+
+#include <asm/unistd.h>
+#include <unistd.h>
+#ifndef __NR_gettid
+#endif
+static inline int _gettid_(void)
+{
+	return syscall(__NR_gettid);
+}
+
+#include <sys/time.h>
+static inline unsigned int GET_MSEC(void)
+{
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	return tv.tv_sec * 1000 + tv.tv_usec / 1000;
+}
+#define INIT_MSEC(v) v = GET_MSEC()
+#define CALC_MSEC(v) v = GET_MSEC() - (v)
+static inline unsigned int GET_USEC(void)
+{
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	return tv.tv_sec * 1000000 + tv.tv_usec;
+}
+#define INIT_USEC(v) v = GET_USEC()
+#define CALC_USEC(v) v = GET_USEC() - (v)
+
+__END_DECLS
+
+#include <string>
+#include <stdarg.h>
+#include <cstring>
+#include <stdio.h>
+#include <stdlib.h>
+#include <log4cplus/logger.h>
+#include <log4cplus/configurator.h>
+#include <log4cplus/helpers/stringhelper.h>
+#include <log4cplus/loggingmacros.h>
+#include <log4cplus/asyncappender.h>
+
+using namespace std;
+using namespace log4cplus;
+using namespace log4cplus::helpers;
+
+static Logger logger = Logger::getRoot();
+
+#define LOG4CPLUS_CONF_FILE "/etc/dtc/log4cplus.conf"
+
+/*********************************************
+**TRACE：调试应用的详细步骤
+**DEBUG：算法关键部分的相关信息
+**INFO： 应用的内部状态信息
+**WARN： 可以避免的内部状态信息
+**ERROR：发生了错误，且应用程序知道如何处理它
+**FATAL：发生了不可逆转的错误，程序无法继续运行
+ **********************************************/
+#define log4cplus_trace(fmt, args...)                                          \
+	write_log(logger, 1, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_debug(fmt, args...)                                          \
+	write_log(logger, 2, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_info(fmt, args...)                                           \
+	write_log(logger, 3, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_warning(fmt, args...)                                        \
+	write_log(logger, 4, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_error(fmt, args...)                                          \
+	write_log(logger, 5, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_fatal(fmt, args...)                                          \
+	write_log(logger, 6, __FILE__, __FUNCTION__, __LINE__, fmt, ##args)
+#define log4cplus_bare(lvl, fmt, args...)                                      \
+	write_log(logger, lvl, NULL, NULL, 0, fmt, ##args)
+
+extern void write_log(Logger, int, const char *, const char *, int,
+		      const char *, ...);
+
+extern void init_log4cplus();
+#endif