64 pytest add test cases at pytest (#65)

* Tests: Generating new test conf yaml file.

* tests: build dtcd at docker.

* Update ubuntu-latest&gcc-4.9.yml

* Update ubuntu-latest&gcc-4.9.yml

* Update ubuntu-latest&gcc-4.9.yml

* Update ubuntu-latest&gcc-4.9.yml

* Update ubuntu-latest&gcc-4.9.yml

* Update ubuntu-latest&gcc-4.9.yml

* tests: add sql at testing.

* fix: compatible for different mysql version.

* tests: dtcd CacheOnly select case.

* docker: add pymysql pip lib.

* docker: fix if error.

.github/workflows/ubuntu-latest&gcc-4.9.yml

@@ -46,6 +46,7 @@ jobs:
         sudo apt-get install -y make zlib1g zlib1g-dev bzip2 liblz4-dev libasan0 openssl libmxml-dev
         python -m pip install --upgrade pip
         pip install pytest
+        pip install pymysql
       
     - name: cmake project
       run: |
@@ -73,7 +74,7 @@ jobs:
         cp build/src/agent-watchdog/agent-watchdog dockerfiles/devel/
         cp build/src/complex/async-connector dockerfiles/devel/
         cp build/src/connector/connector dockerfiles/devel/
-        cp build/src/data_lifecycle/data_lifecycle_manager dockerfiles/devel/
+        cp build/src/data_lifecycle/data-lifecycle-manager dockerfiles/devel/
         cp build/src/hwcserver/hwcserver dockerfiles/devel/
 
         cd dockerfiles/devel/
@@ -81,6 +82,11 @@ jobs:
         docker build -t ${{ secrets.DOCKER_USERNAME }}/devel:CacheOnly .
         docker push ${{ secrets.DOCKER_USERNAME }}/devel:CacheOnly
 
+    - name: Run devel CacheOnly docker
+      if: ${{ env.secret_account != '' }}
+      run: |
+        docker run --name CacheOnly -p 20015:20015 -d ${{ secrets.DOCKER_USERNAME }}/devel:CacheOnly /usr/local/dtc/dtcd -d
+
     - name: publish devel Cache with Database docker
       if: ${{ env.secret_account != '' }}
       run: |
@@ -110,7 +116,10 @@ jobs:
         mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "show tables;"
 
     - name: Testing CacheOnly
-
+      if: ${{ env.secret_account != '' }}
       run: |
         cd ${{github.workspace}}/tests
-        pytest
+        docker ps -a
+        mysql -h127.0.0.1 -P20015 -uroot -proot -e "insert into opensource(uid, name) values(1, 'hello') where uid = 1;"
+        mysql -h127.0.0.1 -P20015 -uroot -proot -e "select uid, name from opensouce where uid = 1;"
+        pytest

dockerfiles/devel/dockerfile

@@ -4,7 +4,7 @@ ARG basepath=/usr/local/dtc
 ARG confpath=/etc/dtc
 ARG logpath=/var/log/dtc
 
-RUN mkdir -p $basepath/bin
+RUN mkdir -p $basepath
 RUN mkdir -p $basepath/data
 RUN mkdir -p $basepath/stat
 RUN mkdir -p $basepath/log
@@ -12,7 +12,12 @@ RUN mkdir -p $confpath
 RUN mkdir -p $logpath
 
 COPY dtcd $basepath/dtcd
+COPY dtcagent $basepath/dtcagent
+COPY agent-watchdog $basepath/agent-watchdog
+COPY async-connector $basepath/async-connector
+COPY connector $basepath/connector
+COPY data-lifecycle-manager $basepath/data-lifecycle-manager
+COPY hwcserver $basepath/hwcserver
+
 COPY dtc.yaml $confpath/dtc.yaml
 COPY log4cplus.conf $confpath/log4cplus.conf
-
-CMD ["/usr/local/dtc/dtcd", "-d"]

dockerfiles/devel/dtc.cacheonly.yaml

@@ -1,18 +1,18 @@
-cache:
-   LOG_LEVEL: debug
-   BIND_ADDR: '*:20015/tcp'
-   DTCID: 20015
-   MAX_USE_MEM_MB: 100
-   DTC_MODE: 1 #1: cache only, 0: database in addition.
-vhot:
-   addr: 127.0.0.1:3307
-   username: root
-   password: root
-   database: sharding
-data_lifecycle:
-   SingleQueryCount: 10
-   DataSQLRule: 'status = 0'
-   OperateTimeRule: '00 01 * * * ?'
-   LifeCycleTableName: 'data_lifecycle_table'
-   HotDBName: 'HOT'
-   ColdDBName: 'COLD'
+# 
+# DTC configure file. v2
+# Cache Only Test cases.
+#
+props:
+  log.level: debug
+  listener.port.dtc: 20015
+  shm.mem.size: 100 #MB
+
+primary:
+  table: &table opensource
+  cache:
+    field:
+      - {name: &key uid, type: signed, size: 4}
+      - {name: name, type: string, size: 50}
+      - {name: city, type: string, size: 50}
+      - {name: sex, type: signed, size: 4}
+      - {name: age, type: signed, size: 4}

src/complex/transaction_task.cc

@@ -84,7 +84,7 @@ void encode_mysql_header(CBufferChain *r, int len, uint8_t pkt_num)
 	*(r->data + 3) = pkt_num;
 }
 
-CBufferChain *encode_eof(CBufferChain *bc, uint8_t pkt_nr)
+CBufferChain *encode_eof(CBufferChain *bc, uint8_t &pkt_nr)
 {
 	CBufferChain *nbc = bc;
 	my_result_set_eof eof;
@@ -104,7 +104,7 @@ CBufferChain *encode_eof(CBufferChain *bc, uint8_t pkt_nr)
 	memcpy(r->data + sizeof(MYSQL_HEADER_SIZE), &eof, sizeof(eof));
 	r->usedBytes = sizeof(MYSQL_HEADER_SIZE) + sizeof(eof);
 	r->nextBuffer = NULL;
-	encode_mysql_header(r, sizeof(eof), pkt_nr);
+	encode_mysql_header(r, sizeof(eof), pkt_nr++);
 
 	nbc->nextBuffer = r;
 	nbc = nbc->nextBuffer;
@@ -469,10 +469,13 @@ CBufferChain* TransactionTask::encode_mysql_protocol(CTaskRequest *request)
 	if (!pos)
 		return NULL;
 
-	//Different MYSQL Version.
-	//pos = encode_eof(pos, ++pkt_nr);
-	//if (!pos)
-	//	return NULL;
+	if(request->get_eof_packet_new() == false)
+	{
+		//Different MYSQL Version.
+		pos = encode_eof(pos, pkt_nr);
+		if (!pos)
+			return NULL;
+	}
 
 	CBufferChain *prow = encode_row_data(m_DBConn, pos, pkt_nr);
 	if (prow) {

src/data_lifecycle/CMakeLists.txt

@@ -59,9 +59,9 @@ LINK_LIBRARIES(libmysqlclient.a)
 
 ADD_DEFINITIONS ("-g -fPIC -fpermissive -std=gnu++11 -DOMN_PLATFORM_UNIX -Wl,--no-undefined -Xlinker -zmuldefs")
 ADD_DEFINITIONS(-Wno-builtin-macro-redefined)
-ADD_EXECUTABLE (data_lifecycle_manager ${SRC_LIST} ../connector/database_connection.cc)
-TARGET_LINK_LIBRARIES(data_lifecycle_manager libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a libyaml-cpp.a libz64.a libsqlparser.a libmysqlclient.a libmxml.a)
-redefine_file_macro(data_lifecycle_manager)
+ADD_EXECUTABLE (data-lifecycle-manager ${SRC_LIST} ../connector/database_connection.cc)
+TARGET_LINK_LIBRARIES(data-lifecycle-manager libdtcd.a libdaemons.a libstat.a libcommon.a liblog4cplus.a libyaml-cpp.a libz64.a libsqlparser.a libmysqlclient.a libmxml.a)
+redefine_file_macro(data-lifecycle-manager)
 
 if(jdtestOpen)
     cmake_minimum_required(VERSION 2.8.8)

src/libs/common/my/my_request.cc

@@ -52,15 +52,22 @@ bool MyRequest::do_mysql_protocol_parse()
 
 	input_packet_length --;
 	p++;
-
+	int count = 0;
 	if (*p == 0x0) {
 		p++;
 		input_packet_length--;
+		count++;
 	}
 
 	if (*p == 0x01) {
 		p++;
 		input_packet_length--;
+		count++;
+	}
+	if(count == 2)
+	{
+		log4cplus_debug("new version query request.");
+		eof_packet_new = true;
 	}
 	m_sql.assign(p, input_packet_length);
 	log4cplus_debug("sql: \"%s\"", m_sql.c_str());

src/libs/common/my/my_request.h

@@ -25,7 +25,7 @@
 
 class MyRequest {
     public:
-	MyRequest() : raw(NULL), raw_len(0), pkt_nr(0)
+	MyRequest() : raw(NULL), raw_len(0), pkt_nr(0), eof_packet_new(false)
 	{
 	}
 
@@ -107,6 +107,7 @@ class MyRequest {
 	int raw_len;
 	std::string m_sql;
 	hsql::SQLParserResult m_result;
+	bool eof_packet_new;
 	uint8_t pkt_nr;
 };
 

src/libs/common/packet/packet_server.cc

@@ -905,7 +905,7 @@ struct my_result_set_eof {
 };
 #pragma pack()
 
-BufferChain *encode_eof(BufferChain *bc, uint8_t pkt_nr)
+BufferChain *encode_eof(BufferChain *bc, uint8_t &pkt_nr)
 {
 	BufferChain *nbc = bc;
 	my_result_set_eof eof;
@@ -925,7 +925,7 @@ BufferChain *encode_eof(BufferChain *bc, uint8_t pkt_nr)
 	memcpy(r->data + sizeof(MYSQL_HEADER_SIZE), &eof, sizeof(eof));
 	r->usedBytes = sizeof(MYSQL_HEADER_SIZE) + sizeof(eof);
 	r->nextBuffer = NULL;
-	encode_mysql_header(r, sizeof(eof), pkt_nr);
+	encode_mysql_header(r, sizeof(eof), pkt_nr++);
 
 	nbc->nextBuffer = r;
 	nbc = nbc->nextBuffer;
@@ -1122,10 +1122,15 @@ BufferChain *Packet::encode_mysql_protocol(DtcJob *job)
 	pos = encode_field_def(job, bc, pkt_nr);
 	if (!pos)
 		return NULL;
-	//Different MYSQL Version.
-	//pos = encode_eof(pos, ++pkt_nr);
-	//if (!pos)
-	//	return NULL;
+
+	if(job->mr.eof_packet_new == false)
+	{
+		//Old MYSQL Version.
+		pos = encode_eof(pos, pkt_nr);
+		if (!pos)
+			return NULL;
+	}
+
 	BufferChain *prow = encode_row_data(job, pos, pkt_nr);
 	if (prow) {
 		pos = prow;
@@ -1436,7 +1441,7 @@ int Packet::encode_result_v2(DtcJob &job, int mtu, uint32_t ts)
 	if(bok == false)
 	{
 		nrp = 1 /*fields count info*/ +
-			job.mr.get_need_array().size() /*fields def*/ + 0 /*eof*/ +
+			job.mr.get_need_array().size() /*fields def*/ + (job.mr.eof_packet_new ? 0 : 1) /*eof*/ +
 			(job.result ? job.result->total_rows() : 0) /*row data*/ +
 			1 /*eof*/;
 	}
@@ -1567,7 +1572,7 @@ int Packet::encode_result_mysql(DtcJob &job, int mtu, uint32_t ts)
 	if(bok == false)
 	{
 		nrp = 1 /*fields count info*/ +
-			job.mr.get_need_array().size() /*fields def*/ + 0 /*eof*/ +
+			job.mr.get_need_array().size() /*fields def*/ + (job.mr.eof_packet_new ? 0 : 1) /*eof*/ +
 			(job.result ? job.result->total_rows() : 0) /*row data*/ +
 			1 /*eof*/;
 	}

src/libs/net/task_request.cc

@@ -136,16 +136,22 @@ int CTaskRequest::do_mysql_protocol_parse()
 
 	input_packet_length --;
 	p++;
+	int count = 0;
 
 	if (*p == 0x0) {
 		p++;
 		input_packet_length--;
+		count++;
 	}
 
 	if (*p == 0x01) {
 		p++;
 		input_packet_length--;
+		count++;
 	}
+	if(count == 2)
+		eof_packet_new = true;
+		
 	m_sql.assign(p, input_packet_length);
 	log4cplus_debug("sql: \"%s\"", m_sql.c_str());
 

src/libs/net/task_request.h

@@ -69,6 +69,7 @@ public:
 		resourceId(0),
 		resourceSeq(0),
 		dtc_header_id(0),
+		eof_packet_new(false),
 		cb(NULL), layer(0)
 	{
 	};
@@ -116,6 +117,7 @@ private:
 	std::string result;
 	uint32_t seq_number;
 	uint16_t request_cmd;
+	bool eof_packet_new;
 	std::string m_sql;
 	std::string dbname;
 	uint8_t mysql_seq_id;
@@ -132,6 +134,8 @@ public:
 	CClientAgent * OwnerClient();
 	void ClearOwnerClient();
 
+	bool get_eof_packet_new() {return eof_packet_new;}
+
 	int DecodeAgentRequest();
 	inline void SaveRecvedResult(char * buff, int len, int pktcnt)
 	{

tests/test_dtcd_cache_only_.py

@@ -0,0 +1,16 @@
+import pytest
+import pymysql
+
+def test_select():
+    db = pymysql.connect(host='127.0.0.1', port=20015, user='test', password='test', database='test')
+    cursor = db.cursor()
+    sql = "select uid, name from opensource where uid = 1"
+    cursor.execute(sql)
+    results = cursor.fetchall()
+    for row in results:
+        uid = row[0]
+        name = row[1]
+        print("uid=%s, name=%s" % (uid, name))
+        assert uid == 1
+        assert name == "hello"
+    db.close()