update

Signed-off-by: shewer <[email protected]>

Author: shewer

english_conv.py

@@ -16,89 +16,91 @@
 import leveldb
 import luadata
 import re
-def load_csv(csvf):
-    l = []
-    with open(csvf, newline='') as csvfile:
-        rows= csv.DictReader(csvfile)
-        for row in rows:
-            if row['word'][0] == '#':
-                continue
-            if row['phonetic'] != "":
-                row['phonetic'] = '[' + row['phonetic'] +  ']'
-            l.append(row)
-
-    return l
-
-def load_text(textf):
-    l=[]
-    with open(textf,newline='') as textfile:
-        for line in textfile.readlines():
-            if line[0] == '#':
-                continue
-            row = {}
-            word,data = line.split('\t',1)
-            row['word']=word
-            ll= data.split(';',1)
-            if len(ll)>1:
-                row['phonetic'] = ll[0]
-                row['translation']= ll[1]
-            else:
-                row['translation']= ll[0]
-            l.append(row )
-
-    return l
-
-def write_chunk(filename,rows):
-    with open(filename,'w') as fn:
-        fn.write('return {\n')
-        for row in rows:
-            value= luadata.serialize(row) #.replace('\\n','\n').replace('\\r','\r')
-            fn.write(value + ',\n')
-        fn.write('}\n')
-
-def write_leveldb(filename,rows):
-    db = leveldb.LevelDB(filename,create_if_missing=True)
-    for row in rows:
-        key = row['word']
-        if not key:
-            continue
-
-        if re.findall('[A-Z]',key):
-            key = key.lower() + '\t' + key
-
-        value= luadata.serialize(row).replace('\\n','\n').replace('\\r','\r')
-        db.Put(key.encode('utf-8'),value.encode('utf-8'))
-
-
-
-
-
-
-def main(fn,fmt):
+def __convert_dict_(rec):
+    if rec[0] == '#':
+        return
+    res= {}
+    res['word'],data = line.strip().split('\t',1)
+    ll= data.split(';',1)
+    if len(ll)>1:
+       res['phonetic'] = ll[0]
+       res['translation']= " " + ll[1]
+    else:
+       res['phonetic'] = ''
+       res['translation']= " " + ll[0]
+    return res
+
+def get_key_str(dict):
+    key = dict and dict.get('word')
+    if not key or len(key) ==0 or  key[0] == '#':
+        return
+    return  key  if key.islower() else key.lower() + "\t" + key
+
+#  rec:  type   dict  or  text format
+def convert_chunk(rec):
+    if isinstance(rec, str):
+        rec = __convert_dicti_(rce)
+    if not rec:
+        return
+
+    rec['translation'] = re.sub(r'^(\w+\.\s)',' \\1', rec['translation'])
+    rec['definition'] = re.sub(r'^(\w+\.\s)',' \\1', rec['definition'])
+                                
+    key = get_key_str(rec)
+    if not key:
+        return
+    tmp= { k: re.sub(r'(\\[rn])+','\n',v) for k,v in rec.items()}
+    if len(tmp['phonetic']) > 0:
+           tmp['phonetic'] = ('[{}]').format(tmp['phonetic'])
+    return key, luadata.serialize(tmp).replace('\\\n','\\n')
+
+class LuaChunk:
+  def __init__(self, name):
+      self.__name = name
+      self.__fn = open(self.__name,'w')
+      self.__status = True
+      self.__fn.write("return {\n")
+  def status(self):
+      return self.__status
+  def name(self):
+      return self.__name
+  def __del__(self):
+      if self.__status:
+          self.close()
+  def Flush(self):
+      if self.__status:
+          self.__fn.flush()
+  def close(self):
+      if self.__status:
+          print('close file: ' + self.__name )
+          self.__fn.write("}\n")
+          self.__fn.close()
+          self.__status = False
+  def Put(self,key, value):
+      if self.__status:
+          self.__fn.write( ('{},\n').format(value.decode()))
+
+
+def __main(fname,fmt):
     """TODO: Docstring for main.
     :returns: TODO
     python conv_file.py  file.[txt|csv]  [leveldb|chunk] -- default: luac compile to chunk_bin
 
     """
     try:
-
-        f,ext = os.path.splitext(fn)
-        rows = ext == ".csv" and  load_csv(fn) or load_text(fn)
-        if fmt== 'leveldb':
-            write_leveldb(f,rows)
-        elif fmt == 'chunk':
-            write_chunk(f + '.txtl', rows)
-        else:
-            write_chunk(f + '.txtll',rows)
-            print("compile to bin ")
-            os.system( 'luac -o '+ f + ".txtl " + f + ".txtll && rm " + f +".txtll" )
-
-
+        f,ext = os.path.splitext(fname)
+        db = leveldb.LevelDB(f + ".userdb") if fmt =='leveldb' else LuaChunk(f + '.txtl')
+        with open(fname) as fnode:
+           inf = csv.DictReader(fnode) if ext=='.csv' else fnode
+           for row in inf:
+               key_str, chunk_str=convert_chunk(row)
+               if  key_str and chunk_str :
+                   db.Put(key_str.encode('utf-8'),chunk_str.encode('utf-8'))
+           del db
     except ValueError as ve:
         return str(ve)
 
 
-
 if __name__ == '__main__' :
 
     print(sys.argv,len(sys.argv))
@@ -107,5 +109,5 @@ def main(fn,fmt):
     else:
         fn= sys.argv[1]
         fmt= len(sys.argv)>2 and sys.argv[2] 
-        sys.exit(main(fn,fmt))
+        sys.exit(__main(fn,fmt))
 

lua/component/stroke_count.lua

@@ -69,6 +69,7 @@ end
 
 function M.func(input,env)
   local context=env.engine.context
+  require("croissant.debugger")()
   for cand in input:iter() do
     cand.comment = cand.comment ..  strock_count_to_str(env.reversedb, cand.text,env.str_fmt)
     yield(cand)