207 supporting string key hash in dtc (#208)

* supporting single db and sharding table.

* fix dtc bugs for sharding db.

* utils update.

.github/workflows/release.yml

@@ -71,7 +71,7 @@ jobs:
           chmod +x dtc/bin/dtc.sh
           chmod +x dtc/bin/agent.sh
 
-          wget "https://storage.jd.com/bq-install/shardingsphere/sharding-ff0cc2.tgz" -O dtc/sharding.tar.gz
+          wget "https://storage.jd.com/bq-install/shardingsphere/sharding-851d4a.tgz" -O dtc/sharding.tar.gz
           tar -zxvf dtc/sharding.tar.gz -C dtc
           rm dtc/sharding.tar.gz
           cp ../dockerfiles/devel/mysql-connector-java-5.1.49.jar dtc/sharding/lib/mysql-connector-java-5.1.49.jar

conf/dtc.yaml

@@ -6,9 +6,6 @@ props:
   listener.port.dtc: 20015
   listener.port.extension: 2002
   shm.mem.size: 100 #MB
-  #hash.custom: yes/no/enable/disable
-  #hash.custom.module: ../lib/key-hash.so
-  #hash.custom.functon: StringHash(1.128)
 
 data_lifecycle:
   single.query.count: 10

dockerfiles/devel/dockerfile

@@ -29,7 +29,7 @@ COPY libdtcapi.so /usr/local/lib/libdtcapi.so
 RUN apt update
 RUN apt install -y iputils-ping net-tools wget default-jre
 
-RUN wget "https://storage.jd.com/bq-install/shardingsphere/sharding-ff0cc2.tgz" -O $basepath/sharding.tar.gz
+RUN wget "https://storage.jd.com/bq-install/shardingsphere/sharding-851d4a.tgz" -O $basepath/sharding.tar.gz
 RUN tar -zxvf $basepath/sharding.tar.gz -C $basepath
 #RUN mv $basepath/apache-shardingsphere-5.1.2-shardingsphere-proxy-bin $basepath/sharding
 COPY mysql-connector-java-5.1.49.jar $basepath/sharding/lib/mysql-connector-java-5.1.49.jar

src/libs/common/config/dbconfig.cc

@@ -36,31 +36,36 @@
 
 static int ParseDbLine(const char *buf, uint16_t *dbIdx)
 {
-    char *p = (char *)buf; // discard const
-
-    int n = 0;
-    while (*p) {
-        if (!isdigit(p[0]))
-            break;
-        int begin, end;
-        begin = strtol(p, &p, 0);
-        if (*p != '-')
-            end = begin;
-        else {
-            p++;
-            if (!isdigit(p[0]))
-                break;
-            end = strtol(p, &p, 0);
-        }
-        while (begin <= end)
-            dbIdx[n++] = begin++;
-
-        if (p[0] != ',')
-            break;
-        else
-            p++;
-    }
-    return n;
+    char *p = (char *)buf;	// discard const
+	if(*p++ != '[')
+		return -1;
+
+	int n = 0;
+	while(*p)
+	{
+		if(!isdigit(p[0]))
+			break;
+		int begin, end;
+		begin = strtol(p, &p, 0);
+		if(*p!='-' )
+			end = begin;
+		else {
+			p++;
+			if(!isdigit(p[0]))
+				break;
+			end = strtol(p, &p, 0);
+		}
+		while(begin <= end)
+			dbIdx[n++] = begin++;
+
+		if(p[0]!=',')
+			break;
+		else
+		    p++;
+	}
+	if(p[0] != ']')
+		return -1;
+	return n;	
 }
 
 /* 前置空格已经过滤了 */
@@ -126,7 +131,8 @@ bool FieldConfig::is_same(FieldConfig *field)
 
 int DbConfig::load_key_hash(DTCConfig *raw)
 {
-    log4cplus_debug("tryto load key-hash plugin");
+    log4cplus_debug("try to load key-hash plugin");
+    bool hashopen = false;
 
     /* init. */
     keyHashConfig.keyHashEnable = 0;
@@ -135,27 +141,26 @@ int DbConfig::load_key_hash(DTCConfig *raw)
     keyHashConfig.keyHashRightBegin = 0;
 
     /* not enable key-hash */
-    std::string hashopen = raw->get_config_node()["props"]["hash.custom"].as<std::string>();
-    if (str2int(hashopen.c_str(), 0) == 0) {
+    if(raw->get_config_node()["primary"]["hot"])
+    {
+        if(raw->get_config_node()["primary"]["hot"]["sharding"] && 
+            (raw->get_config_node()["primary"]["cache"]["field"][0]["type"].as<std::string>() == "string" || 
+            raw->get_config_node()["primary"]["cache"]["field"][0]["type"].as<std::string>() == "binary"))
+            {
+                hashopen = true;
+            }
+    }
+
+    if (!hashopen) {
         log4cplus_debug("key-hash plugin disable");
         return 0;
     }
 
     /* read key_hash_module */
-    std::string so = raw->get_config_node()["props"]["hash.custom.module"].as<std::string>();
-    if (so.length() == 0) {
-        log4cplus_info(
-            "not set key-hash plugin name, use default value");
-        so = DEFAULT_KEY_HASH_SO_NAME;
-    }
+    std::string so = DEFAULT_KEY_HASH_SO_NAME;
 
     /* read key_hash_function */
-    std::string var =
-        raw->get_config_node()["props"]["hash.custom.functon"].as<std::string>();
-    if (var.length() == 0) {
-        log4cplus_error("not set key-hash plugin function name");
-        var = DEFAULT_KEY_HASH_FUNCTION;
-    }
+    std::string var = DEFAULT_KEY_HASH_FUNCTION;
 
     char *fun = 0;
     int isfunalloc = 0;
@@ -323,11 +328,6 @@ int DbConfig::convert_case_sensitivity(
     return i;
 }
 
-int get_db_machine_count()
-{
-    return 1;
-}
-
 std::string get_merge_string(YAML::Node node)
 {
     std::string str = "";
@@ -371,7 +371,7 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
     //DB section
     if(dtc_config["primary"][layer])    //cache.datasource mode
     {
-        machineCnt = get_db_machine_count();
+        machineCnt = dtc_config["primary"][layer]["real"].size();
         if (machineCnt <= 0) {
             log4cplus_error("%s", "invalid server_count");
             return -1;
@@ -385,7 +385,7 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
     if(dtc_config["primary"][layer]) //cache.datasource mode
     {
         YAML::Node node = dtc_config["primary"][layer]["real"];
-        if(node.size() == 1) //single db
+        if(node.size() == 1 && dtc_config["primary"][layer]["real"][0]["db"].IsScalar()) //single db
         {
             if(dtc_config["primary"][layer]["sharding"] && 
                 dtc_config["primary"][layer]["sharding"]["table"]["last"].as<int>() - dtc_config["primary"][layer]["sharding"]["table"]["start"].as<int>() + 1 > 1)
@@ -393,7 +393,7 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
             else
                 depoly = SINGLE;
         }
-        else if(node.size() > 1) //multi db
+        else if(node.size() >= 1) //multi db
         {
             if(dtc_config["primary"][layer]["sharding"] && 
             dtc_config["primary"][layer]["sharding"]["table"]["last"].as<int>() - dtc_config["primary"][layer]["sharding"]["table"]["start"].as<int>() + 1 > 1)
@@ -432,10 +432,9 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
     dstype = 0;
     checkTable = 0;
 
-    //TODO: string key supporting.
     // key-hash dll
-    //if (load_key_hash(raw) != 0)
-    //    return -1;
+    if (load_key_hash(raw) != 0)
+        return -1;
 
     if(dtc_config["primary"][layer])
     {
@@ -469,7 +468,15 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
             }
         }
 
-        database_max_count = dtc_config["primary"][layer]["real"].size();
+        if(dtc_config["primary"][layer]["real"][0]["db"].IsScalar())
+        {
+            database_max_count = 1;
+        }
+        else
+        {
+            database_max_count = dtc_config["primary"][layer]["real"][0]["db"]["last"].as<int>() - dtc_config["primary"][layer]["real"][0]["db"]["start"].as<int>() + 1;
+        }
+        
         if (database_max_count < 0 || database_max_count > 10000) {
             log4cplus_error("%s", "invalid [DATABASE_CONF].DbMax");
             return -1;
@@ -612,8 +619,13 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
         if(dtc_config["primary"][layer]["real"][i]["db"].IsScalar())
             m->dbCnt = 1;
         else
-            m->dbCnt = dtc_config["primary"][layer]["real"][i]["db"]["last"].as<int>() - 
-                        dtc_config["primary"][layer]["real"][i]["db"]["start"].as<int>() + 1;
+        {
+            char szIdx[64] = {0};
+            sprintf(szIdx, "[%d-%d]", dtc_config["primary"][layer]["real"][i]["db"]["start"].as<int>(), 
+                    dtc_config["primary"][layer]["real"][i]["db"]["last"].as<int>());
+            m->dbCnt = ParseDbLine (szIdx, m->dbIdx);
+        }
+
         for (int j = 0; j < m->dbCnt; j++) {
             if (m->dbIdx[j] >= database_max_count) {
                 log4cplus_error(

src/libs/common/table/table_def.cc

@@ -236,7 +236,6 @@ int DTCTableDefinition::add_field(int id, const char *name, uint8_t type,
 		usedFields++;
 	if (id != 0) {
 		rawFields[id] = id;
-		log4cplus_info("rawFields[id]:%d" , rawFields[id]);
 	}
 	return 0;
 }

src/utils/conf-gen-utils.cc

@@ -228,10 +228,8 @@ int yaml_dump_sharding_rule(FILE *fp, std::vector<YAML::Node> vec)
                 fprintf(fp, "          shardingAlgorithmName: %s\n", szname);
 
                 char sztemp[1024] = {0};
-                sprintf(sztemp, "$%s{(%s%%%d)}", 
-                    get_merge_string(node["real"][0]["db"]["prefix"]).c_str(),
-                    key.c_str(),
-                    dbcount);
+                sprintf(sztemp, "        sharding-count: %d\n        sharding-divide: %d\n", 
+                    dbcount, 1);
                 algorithm[szname] = sztemp;
             }
             
@@ -242,17 +240,11 @@ int yaml_dump_sharding_rule(FILE *fp, std::vector<YAML::Node> vec)
             sprintf(szname, "%s_tb_inline", tbname.c_str());
             fprintf(fp, "          shardingAlgorithmName: %s\n", szname);
             char sztemp[1024] = {0};
-            sprintf(sztemp, "%s${((%s/%d).longValue()%%%d)}", 
-                get_merge_string(node["sharding"]["table"]["prefix"]).c_str(),
-                key.c_str(),
-                dbcount,
+            sprintf(sztemp, "        sharding-count: %d\n        sharding-divide: %d\n", 
                 node["sharding"]["table"]["last"].as<int>() -
-                node["sharding"]["table"]["start"].as<int>() + 1);
+                node["sharding"]["table"]["start"].as<int>() + 1,
+                dbcount);
             algorithm[szname] = sztemp;
-
-            fprintf(fp, "      keyGenerateStrategy:\n");
-            fprintf(fp, "        column: %s\n", key.c_str());
-            fprintf(fp, "        keyGeneratorName: snowflake\n");
         }
     }
 
@@ -263,9 +255,9 @@ int yaml_dump_sharding_rule(FILE *fp, std::vector<YAML::Node> vec)
         for (it = algorithm.begin(); it != algorithm.end(); it++) 
         {
             fprintf(fp, "    %s:\n", (*it).first.c_str());
-            fprintf(fp, "      type: INLINE\n");
+            fprintf(fp, "      type: HASH_MOD\n");
             fprintf(fp, "      props:\n");
-            fprintf(fp, "        algorithm-expression: %s\n", (*it).second.c_str());
+            fprintf(fp, "%s\n", (*it).second.c_str());
         }
     }