dtc/.github/workflows/build-and-test.yml

name: build-and-test

on: push
    
env:
  CC: gcc-4.9
  CXX: g++-4.9
  secret_account: ${{ secrets.DOCKER_USERNAME }}

jobs:        
  building:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      
      - name: Install gcc env
        run: |
          echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" | sudo tee -a /etc/apt/sources.list
          echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" | sudo tee -a /etc/apt/sources.list
          sudo apt update
          sudo apt install gcc-4.9 g++-4.9
        shell: bash
        
      - name: Install dependency
        run: |
          sudo apt-get install -y make zlib1g zlib1g-dev bzip2 liblz4-dev libasan0 openssl libmxml-dev wget

      - name: update hsql
        run: |
          sh ${{github.workspace}}/script/download.sh
          tar -zxvf hsql.tar.gz
          cd hsql
          cp -rf include/ ${{github.workspace}}/src/libs/hsql/
          cp -f libs/libsqlparser.so ${{github.workspace}}/src/libs/hsql/libs/libsqlparser.so
          cp -f libs/libsqlparser.a ${{github.workspace}}/src/libs/hsql/libs/libsqlparser.a
    
      - name: cmake project
        run: |
          cd ${{github.workspace}}
          mkdir build
          cd build
          cmake ../
          make

      - name: Login docker hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      
      - name: Publish devel docker
        run: |
          cd ${{github.workspace}}
          cp build/src/core/dtcd dockerfiles/devel/
          cp build/src/agent/dtcagent dockerfiles/devel/
          cp build/src/agent-watchdog/agent-watchdog dockerfiles/devel/
          cp build/src/complex/async-conn dockerfiles/devel/
          cp build/src/connector/connector dockerfiles/devel/
          cp build/src/data_lifecycle/data-lifecycle-manager dockerfiles/devel/
          cp build/src/hwcserver/hwcserver dockerfiles/devel/
          cp build/src/rule/librule.so dockerfiles/devel/
          cp src/libs/hsql/libs/libsqlparser.so dockerfiles/devel/
          cp build/src/utils/conf-gen-utils dockerfiles/devel/
          cp src/libs/api/cc_api/libs/libdtcapi.so dockerfiles/devel/

          cd dockerfiles/devel/

          docker build -t ${{ secrets.DOCKER_USERNAME }}/devel:all .
          docker push ${{ secrets.DOCKER_USERNAME }}/devel:all

          docker build -t ${{ secrets.DOCKER_USERNAME }}/devel:latest .
          docker push ${{ secrets.DOCKER_USERNAME }}/devel:latest

# Core Junction
  CORE:
    needs: building
    runs-on: ubuntu-20.04
    steps:
      - run: echo "Core Junction."  

# Agent Junction
  AGENT:
    needs: building
    runs-on: ubuntu-20.04
    steps:
      - run: echo "Agent Junction."        

# Layered Junction
  LAYERED:
    needs: building
    runs-on: ubuntu-20.04
    steps:
      - run: echo "Layered Junction."          

# Testing Cache Only
  test-dtc-cache-only:
    needs: CORE
    runs-on: ubuntu-20.04
    services:
      dtc:
        image: docker.io/kfysck/devel:all
        ports:
          - 20015:20015
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name dtc
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -c
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.cacheonly.yaml /usr/local/etc/dtc.yaml

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          docker logs dtc
          cd ${{github.workspace}}/tests
          pytest test_dtcd_cache_only.py     

# Testing Cache Only via Agent.
  test-agent-cache-only:
    if: false
    needs: AGENT
    runs-on: ubuntu-20.04
    services:
      dtc:
        image: docker.io/kfysck/devel:all
        ports:
          - 20015:20015
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name dtc
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -c        
      agent:
        image: docker.io/kfysck/devel:all
        ports:
          - 12001:12001
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name agent
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -a
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql
          sudo apt update
          sudo apt install -y net-tools

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.cacheonly.yaml /usr/local/etc/dtc.yaml
          sudo cp -f dockerfiles/devel/agent.single.dtcd.xml /usr/local/etc/agent.xml

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          netstat -ntpl
          docker logs agent
          docker logs dtc
          cd ${{github.workspace}}/tests
          pytest test_agent_cache_only.py               

# Testing Layered Storage.
  test-agent-layered:
    if: false
    needs: LAYERED
    runs-on: ubuntu-20.04
    services:
      mysql:
        image: mysql:5.7
        ports:
          - 3306:3306
        env: 
          MYSQL_ROOT_PASSWORD: 123456
      agent:
        image: docker.io/kfysck/devel:all
        ports:
          - 12001:12001
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name agent
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -ayc
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Init db table
        run: |
          sleep 5s
          mysql -h127.0.0.1 -uroot -p123456 -e "create database layer2;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "source tests/init.s1.sql;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "show tables;"    
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "desc opensource;"  
          mysql -h127.0.0.1 -uroot -p123456 -e "create database layer3;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer3 -e "source tests/init.s1.sql;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer3 -e "show tables;"    
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer3 -e "desc opensource;"            

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.layered.yaml /usr/local/etc/dtc.yaml
          sudo cp -f dockerfiles/devel/agent.layered.xml /usr/local/etc/agent.xml

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          netstat -ntpl
          docker logs agent
          cd ${{github.workspace}}/tests
          echo "show databases;"
          mysql -h127.0.0.1 -P12001 -uroot -p123456 -e "show databases;"
          echo "use layer2;"
          mysql -h127.0.0.1 -P12001 -uroot -p123456 -Dlayer2 -e "use layer2;"
          echo "show tables;"
          mysql -h127.0.0.1 -P12001 -uroot -p123456 -Dlayer2 -e "show tables;"
#          pytest test_agent_cache_only.py           

# Testing Cache with Datasource Scene 1 (Single DB and Single Table)
  test-dtc-ds-S-db-S-table:
    needs: CORE
    runs-on: ubuntu-20.04
    services:
      mysql:
        image: mysql:5.7
        ports:
          - 3306:3306
        env: 
          MYSQL_ROOT_PASSWORD: 123456
      dtc:
        image: docker.io/kfysck/devel:all
        ports:
          - 20015:20015
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name dtc
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -c
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Init db table
        run: |
          sleep 5s
          mysql -h127.0.0.1 -uroot -p123456 -e "create database layer2;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "source tests/init.s1.sql;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "show tables;"    
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "desc opensource;"  

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.dbaddition.s1.yaml /usr/local/etc/dtc.yaml

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          docker logs dtc
          cd ${{github.workspace}}/tests
          pytest test_dtcd_datasource_s1.py               

# Testing Cache with Datasource Scene 1 (Single DB and Single Table) via Agent
  test-agent-ds-S-db-S-table:
    needs: AGENT
    if: false
    runs-on: ubuntu-20.04
    services:
      mysql:
        image: mysql:5.7
        ports:
          - 3306:3306
        env: 
          MYSQL_ROOT_PASSWORD: 123456
      agent:
        image: docker.io/kfysck/devel:all
        ports:
          - 12001:12001
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name agent
        env:
          DTC_BIN: dtcagent       
      dtc:
        image: docker.io/kfysck/devel:all
        ports:
          - 20015:20015
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name dtc
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -c
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Init db table
        run: |
          sleep 5s
          mysql -h127.0.0.1 -uroot -p123456 -e "create database layer2;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "source tests/init.s1.sql;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "show tables;"    
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "desc opensource;"  

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.dbaddition.s1.yaml /usr/local/etc/dtc.yaml
          sudo cp -f dockerfiles/devel/agent.single.dtcd.xml /usr/local/etc/agent.xml          

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          docker logs agent
          docker logs dtc
          cd ${{github.workspace}}/tests
          pytest test_agent_datasource_s1.py                         

# Testing Cache with Datasource Scene 2 (Single DB and Multi Table)
  test-dtc-ds-S-db-M-table:
    needs: CORE 
    runs-on: ubuntu-20.04
    services:
      mysql:
        image: mysql:5.7
        ports:
          - 3306:3306
        env: 
          MYSQL_ROOT_PASSWORD: 123456
      dtc:
        image: docker.io/kfysck/devel:all
        ports:
          - 20015:20015
        volumes:
          - /usr/local/etc:/usr/local/dtc/conf/
        options: --name dtc
        env:
          DTC_BIN: agent-watchdog
          DTC_ARGV: -c
    steps:
      - uses: actions/checkout@v3
      
      - name: Install python dependency
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install pymysql

      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: "3.7"

      - name: Init db table
        run: |
          sleep 5s
          mysql -h127.0.0.1 -uroot -p123456 -e "create database layer2;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "source tests/init.s2.sql;"
          mysql -h127.0.0.1 -uroot -p123456 -Dlayer2 -e "show tables;"    

      - name: Copy conf files.
        run: |
          cd ${{github.workspace}}   
          sudo cp -f conf/log4cplus.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-hwc.conf /usr/local/etc/ 
          sudo cp -f conf/log4cplus-wd.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-agent.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-async.conf /usr/local/etc/
          sudo cp -f conf/log4cplus-life.conf /usr/local/etc/
          sudo cp -f dockerfiles/devel/dtc.dbaddition.s2.yaml /usr/local/etc/dtc.yaml

      - name: Run Testing Cases
        run: |
          sleep 5s
          docker ps -a
          docker logs dtc
          cd ${{github.workspace}}/tests
          pytest test_dtcd_datasource_s2.py