wwsearch/deps/tokenizer/include/Segmenter.h

/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* ***** BEGIN LICENSE BLOCK *****
* Version: GPL 2.0
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License. You should have
* received a copy of the GPL license along with this program; if you
* did not, you can find it at http://www.gnu.org/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is Coreseek.com code.
*
* Copyright (C) 2007-2008. All Rights Reserved.
*
* Author:
*	Li monan <li.monan@gmail.com>
*
* ***** END LICENSE BLOCK ***** */

#ifndef css_Segmenter_h
#define css_Segmenter_h

#include <vector>
#ifdef WIN32
#include <hash_map>
#else
#include <ext/hash_map>
#endif
#include "SegmentPkg.h"
#include "UnigramDict.h"
#include "SynonymsDict.h"
#include "csr_typedefs.h"
#include "freelist.h"
#include "mmthunk.h"

#include <algorithm>
#include <fstream>
#include <iostream>

#include "UnigramCorpusReader.h"

#include "ThesaurusDict.h"

namespace css {
using namespace CRFPP;
#ifdef WIN32
using namespace stdext;
#else
using namespace __gnu_cxx;
#endif

#define CRFDICT_UTF8 1

#define BEGIN_TOKEN "b##b"
#define END_TOKEN "e##e"
#define NUMBER_TOKEN "M"
#define ASCII_TOKEN "E"

#define BEGIN_TOKEN_ID 0
#define END_TOKEN_ID 1
#define NUMBER_TOKEN_ID 2
#define ASCII_TOKEN_ID 3

#define BEGIN_TOKEN_LENGTH 4
#define END_TOKEN_LENGTH 4
#define NUMBER_TOKEN_LENGTH 1
#define ASCII_TOKEN_LENGTH 1

/*
base functor, used to abstract n-gram smoothing algorithm
Design only. not used yet.
*/
template <typename FType>
struct NgramSmoother {
  FType operator()(int L, int R, int Bi, FType Smoothing) const {
    double dTemp = 1.0 / MAX_FREQUENCE;
    return (-1) * log(Smoothing * (1 + L) / (MAX_FREQUENCE + 80000) +
                      (1 - Smoothing) * ((1 - dTemp) * Bi / (1 + L) + dTemp));
    return 0;
  }
  const static int MAX_FREQUENCE = 2079997;
};

/**
Bit flag format:
Bit flag is used in char-type tagging. size = sizeof(char).
x1 x2 x3 x4 x5 x6 x7
x1 x2, the utf-8 char's position token
1 1, the next 2(or 4) char is token-length. (utf-8 data length)
0 0, only current char
0 1, next char
1 0, next 2 char
1 1, more than 3 char, read next 2 byte. this limited a token can not larger
than 64k.
------
[0-80], the standard ascii char,
tag-set:
m: number
e: non CJK char, e.g. English pinyin
t: time.    年号 干支等（此处识别出后，仅加入 oov ，不参与实际分词）
c: CJK char.
s: Symbol e.g. @
w: Sentence seperator.
x: unknown char.
*/

class Segmenter_ConfigObj {
 public:
  u1 merge_number_and_ascii;
  u1 seperate_number_ascii;
  // TODO: compress_space is still unsupported, for spaces can be handled in
  // stopword list.
  u1 compress_space;
  u1 number_and_ascii_joint[512];
  Segmenter_ConfigObj()
      : merge_number_and_ascii(0), seperate_number_ascii(0), compress_space(0) {
    number_and_ascii_joint[0] = 0;
  }
};

class Segmenter {
 public:
  /**
   *  @return 0
   */
  void setBuffer(u1* buf, u4 length);
  const u1* peekToken(u2& aLen, u2& aSymLen, u2 n = 0);
  void popToken(u2 len, u2 n = 0);
  void segNgram(int n) { m_ngram = n; }
  int getOffset();
  u1 isSentenceEnd();
  int isKeyWord(u1* buf, u4 length);
  int getWordWeight(u1* buf, u4 length);

  const char* thesaurus(const char* key, u2 key_len);
  Segmenter();
  ~Segmenter();

 protected:
  const u1* peekKwToken(u2& aLen, u2& aSymLen);
  void popKwToken(u2 len);

 public:
  static int toLowerCpy(const u1* src, u1* det, u2 det_size);

 protected:
  int m_begin_id;
  int m_end_id;
  int m_begin_count;
  int m_end_count;
  int m_ngram;

  ChineseCharTaggerImpl* m_tagger;
  MMThunk m_thunk;
  // static ToLowerImpl* m_lower;
 public:
  UnigramDict* m_unidict;
  UnigramDict* m_kwdict;
  UnigramDict* m_weightdict;
  SynonymsDict* m_symdict;
  ThesaurusDict* m_thesaurus;

  Segmenter_ConfigObj* m_config;
  // mmseg used.
  u1* m_buffer_begin;
  u1* m_buffer_ptr;
  u1* m_buffer_chunk_begin;
  u1* m_buffer_end;
};

} /* End of namespace css */
#endif