create docsim

Author: zhaoyihua

docsim/__init__.py

@@ -0,0 +1,13 @@
+[path]
+train_dir = xxx
+
+
+[segment]
+cut_all = False  # 精确模式
+HMM = False
+customized_dict = ""
+
+
+[method]
+method = bow
+

docsim/config.cfg

@@ -0,0 +1,76 @@
+import os, sys
+import jieba
+from utils import get_config_store
+
+
+
+class Segmenter(object):
+
+    def __init__(self):
+        '''
+	load configuration
+	'''
+        self.segmenter = jieba 
+
+	config  = get_config_store()
+	self.cut_all = config.get("segment", "cut_all")
+	self.HMM     = config.get("segment", "HMM")
+	customized_dict = config.get("segment", "customized_dict")
+        if customized_dict:
+	    load_dict(customized_dict)
+
+
+    def load_dict( filename ):
+        '''
+	load customized dictionary
+	'''
+        if not os.path.isfile( filename ):
+            sys.stderr.write( "cannot find dictionary %s." %filename )
+            return
+	self.segmenter.load_userdict( filename )
+
+    
+    def cut_line(self, input_line, cut_all=False ):
+        '''
+        just cut a single line, and return a list of tokens
+        '''
+        seg_list = self.segmenter.cut( input_line, cut_all=cut_all )
+        return [ x for x in seg_list if x.strip() ]
+
+
+    def cut_one_file( self, filename, cut_all=False ):
+        '''
+        cut all sentence in a file and return a list of lists
+        '''
+        if not os.path.isfile( filename ):
+            sys.stderr.write( "%s doesn't exist." %filename )
+        fin = open( filename, 'r' )
+        doc = []
+        for line in fin:
+            seg_list = cut_line( line, cut_all )
+            if seg_list:
+	        doc.append( seg_list )
+        return doc
+
+    def cut_files( self, file_list ):
+        '''
+        cut documents in a list of files and merge results to a list of lists
+        '''
+        docs = []
+        for filename in file_list:
+            doc = cut_one_file( filename, self.cut_all )
+            docs.extends( doc )
+        return docs
+
+    def cut_folder(self, path ):
+        '''
+	cut all documents in the folder
+	'''
+	if not os.path.isdir( path ):
+	     sys.stderr.write( "%s doesn't exist." %path )
+	docs = []
+	for filename in os.listdir( path ):
+            doc = cut_one_file( filename, self.cut_all )
+            docs.extends( doc )
+        return docs
+	    

docsim/segmenter.py

@@ -0,0 +1,6 @@
+from segmenter import Segmenter
+
+
+seg = Segmenter()
+tokens = seg.cut_line("hello world")
+print '|'.join(tokens)

docsim/segmenter.pyc

@@ -0,0 +1,29 @@
+import sys, os
+from ConfigParser import ConfigParser
+from os.path import join, dirname, abspath
+
+
+CONFIG = None
+
+def relative(*paths):
+    return join(dirname(abspath(__file__)), *paths)
+
+
+def set_config_file(filename):
+    '''
+    Load configuration from external files
+    '''
+    global CONFIG
+    CONFIG = ConfigParser()
+    CONFIG.read(filename)
+
+
+def get_config_store():
+    '''
+    Return configuration
+    '''
+    global CONFIG
+    if CONFIG is None:
+        CONFIG = ConfigParser()
+    CONFIG.read(relative("config.cfg"))
+    return CONFIG

docsim/test.py

@@ -69,3 +69,7 @@ def addItem( title, description, primaryCategoryId, startPrice='0.99',
         pass
     site_elem         = add_elem(item_elem, "Site", site)
 
+    request = ET.tostring(root, 'utf-8')
+    return get_response(oname, request, encoding)
+   
+