程序包 com.publiccms.common.tools

类 SmartcnDictUtils

java.lang.Object

com.publiccms.common.tools.SmartcnDictUtils

public class SmartcnDictUtils
extends Object

原始dict文件工具类 https://github.com/Kerwin23/smartcn-dict

作者:

Kerwin

字段概要

字段

修饰符和类型	字段	说明
static final int	CHAR_NUM_IN_FILE
private static final String	CHARSET_GB2312
static final Map<String,Integer>	defaultDelimiterFreqsMap
static final int	GB2312_CHAR_NUM
static final int	GB2312_FIRST_CHAR
static final String	TYPE_BIGRAM
static final String	TYPE_CORE

构造器概要

构造器

限定符	构造器	说明
private	SmartcnDictUtils()

方法概要

修饰符和类型	方法	说明
static void	create(String filePath, String type, Map<String,Map<String,Integer>> charTFsMap, Map<String,Integer> delimiterFreqsMap)	创建词库文件
private static String	getCCByGB2312Id(int ccid)
private static short	getGB2312Id(char ch)
private static byte[]	intToLEBytes(int i)
private static void	merge(Map<String,Map<String,Integer>> tfsMap, Map<String,Map<String,Integer>> tempTfsMap, Set<String> keys)
static void	mergeTFsMap(Map<String,Map<String,Integer>> tfsMap, Map<String,Integer> source)	增加新的分词
static String	readCnTerm(String src)
private static void	readCoreMemDelimeter(String cc, char[][] termsArray, int[] freqArray, Map<String,Integer> delimiterFreqsMap)
private static void	readCoreMemFromArrays(char[][][] wordItemCharArrayTable, int[][] wordItemFrequencyTable, Map<String,Map<String,Integer>> charTermFreqsMap, Map<String,Integer> delimiterFreqsMap)
private static void	readCoreMemTerms(String cc, char[][] termsArray, int[] freqArray, Map<String,Map<String,Integer>> charTermFreqsMap)
static void	readFromCoreMem(InputStream inputStream, Map<String,Map<String,Integer>> charTermFreqsMap, Map<String,Integer> delimiterFreqsMap)	读取已有词库mem文件
static void	skipWord(Map<String,Map<String,Integer>> tfsMap, List<String> skipWordList)	删除分词
private static Map<String,Map<String,Integer>>	transSource(Map<String,Integer> source)
private static void	writeDelimiters(OutputStream oStream, Map<String,Integer> delimiterFreqsMap)
private static void	writeEmpty(OutputStream oStream)
private static void	writeInt(OutputStream oStream, int i)
private static void	writeTFs(OutputStream oStream, Map<String,Integer> tfs)

从类继承的方法 java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

字段详细资料

CHARSET_GB2312

private static final String CHARSET_GB2312

另请参阅:

• 常量字段值

GB2312_FIRST_CHAR

public static final int GB2312_FIRST_CHAR

另请参阅:

• 常量字段值

GB2312_CHAR_NUM

public static final int GB2312_CHAR_NUM

另请参阅:

• 常量字段值

CHAR_NUM_IN_FILE

public static final int CHAR_NUM_IN_FILE

另请参阅:

• 常量字段值

TYPE_CORE

public static final String TYPE_CORE

另请参阅:

• 常量字段值

TYPE_BIGRAM

public static final String TYPE_BIGRAM

另请参阅:

• 常量字段值

defaultDelimiterFreqsMap

public static final Map<String,Integer> defaultDelimiterFreqsMap

构造器详细资料

SmartcnDictUtils

private SmartcnDictUtils()

方法详细资料

readCnTerm

public static String readCnTerm(String src)
                         throws UnsupportedEncodingException

抛出:

UnsupportedEncodingException

readFromCoreMem

public static void readFromCoreMem(InputStream inputStream,
 Map<String,Map<String,Integer>> charTermFreqsMap,
 Map<String,Integer> delimiterFreqsMap)
                            throws IOException,
ClassNotFoundException

读取已有词库mem文件

参数:

inputStream -

charTermFreqsMap -

delimiterFreqsMap -

抛出:

IOException

ClassNotFoundException

skipWord

public static void skipWord(Map<String,Map<String,Integer>> tfsMap,
 List<String> skipWordList)

删除分词

参数:

tfsMap -

skipWordList -

mergeTFsMap

public static void mergeTFsMap(Map<String,Map<String,Integer>> tfsMap,
 Map<String,Integer> source)

增加新的分词

参数:

tfsMap -

source -

create

public static void create(String filePath,
 String type,
 Map<String,Map<String,Integer>> charTFsMap,
 Map<String,Integer> delimiterFreqsMap)

创建词库文件

参数:

filePath -

type -

charTFsMap -

delimiterFreqsMap -

transSource

private static Map<String,Map<String,Integer>> transSource(Map<String,Integer> source)

readCoreMemFromArrays

private static void readCoreMemFromArrays(char[][][] wordItemCharArrayTable,
 int[][] wordItemFrequencyTable,
 Map<String,Map<String,Integer>> charTermFreqsMap,
 Map<String,Integer> delimiterFreqsMap)

readCoreMemTerms

private static void readCoreMemTerms(String cc,
 char[][] termsArray,
 int[] freqArray,
 Map<String,Map<String,Integer>> charTermFreqsMap)

readCoreMemDelimeter

private static void readCoreMemDelimeter(String cc,
 char[][] termsArray,
 int[] freqArray,
 Map<String,Integer> delimiterFreqsMap)

merge

private static void merge(Map<String,Map<String,Integer>> tfsMap,
 Map<String,Map<String,Integer>> tempTfsMap,
 Set<String> keys)

writeDelimiters

private static void writeDelimiters(OutputStream oStream,
 Map<String,Integer> delimiterFreqsMap)
                             throws IOException

抛出:

IOException

writeTFs

private static void writeTFs(OutputStream oStream,
 Map<String,Integer> tfs)
                      throws IOException

抛出:

IOException

writeEmpty

private static void writeEmpty(OutputStream oStream)
                        throws IOException

抛出:

IOException

writeInt

private static void writeInt(OutputStream oStream,
 int i)
                      throws IOException

抛出:

IOException

intToLEBytes

private static byte[] intToLEBytes(int i)

getCCByGB2312Id

private static String getCCByGB2312Id(int ccid)

getGB2312Id

private static short getGB2312Id(char ch)
                          throws UnsupportedEncodingException

抛出:

UnsupportedEncodingException