modify test case

Author: shuyo

lda/ldacvb0_cpp/ldacvb0/ldacvb0.hpp

@@ -154,14 +154,18 @@ identify and count freaquency of vocabulary
 */
 template <class WORD>
 class Vocabularies {
+	bool uses_stopwords;
 public:
 	std::vector<WORD> vocalist;
 	std::unordered_map<WORD, IdCount> voca;
 
+	Vocabularies() : uses_stopwords(true) {}
+	Vocabularies(bool excludes_stopwords) : uses_stopwords(!excludes_stopwords) {}
+
 	size_t add(const WORD &word) {
 		WORD key(word);
 		normalize(key);
-		if (STOPWORDS.find(key)!=STOPWORDS.end()) return SIZE_MAX;
+		if (uses_stopwords && STOPWORDS.find(key)!=STOPWORDS.end()) return SIZE_MAX;
 		auto x = voca.find(key);
 		if (x != voca.end()) {
 			x->second.count += 1;
@@ -258,10 +262,11 @@ class Documents : public std::vector<Document> {
 
 public:
 	Documents() : N(0), rexword(REXWORD) {
-		// TODO : stop words
 	}
 	Documents(const std::regex &r) : N(0), rexword(r) {
 	}
+	Documents(const std::regex &r, bool excludes_stopwords) : N(0), rexword(r), vocabularies(excludes_stopwords) {
+	}
 
 	bool add(const CHAR* p, const CHAR* end) {
 		std::regex_iterator<const CHAR*> i( p, end, rexword );

lda/ldacvb0_cpp/ldacvb0_test/test.cpp

@@ -18,13 +18,13 @@ CYBOZU_TEST_AUTO(test_vocabularies_for_std_string)
 "	The/at jury/nn further/rbr said/vbd in/in term-end/nn presentments/nns that/cs the/at City/nn-tl Executive/jj-tl Committee/nn-tl ,/, "
 "which/wdt had/hvd over-all/jj charge/nn of/in the/at election/nn ,/, ``/`` deserves/vbz the/at praise/nn and/cc thanks/nns of/in the/at "
 "City/nn-tl of/in-tl Atlanta/np-tl ''/'' for/in the/at manner/nn in/in which/wdt the/at election/nn was/bedz conducted/vbn ./.";
-	cybozu::ldacvb0::Documents<std::string, char> d(cybozu::ldacvb0::REXWORD_WITH_POS);
+	cybozu::ldacvb0::Documents<std::string, char> d(cybozu::ldacvb0::REXWORD_WITH_POS, true);
 	d.add(st);
 	cybozu::ldacvb0::Vocabularies<std::string>& v = d.vocabularies;
-	CYBOZU_TEST_EQUAL(v.size(), 42);
+	CYBOZU_TEST_EQUAL(v.size(), 37);
 	CYBOZU_TEST_EQUAL(v.count("the"), 8);
 	CYBOZU_TEST_EQUAL(v.count("tHE"), 8);
-	CYBOZU_TEST_EQUAL(v.count("in"), 2);
+	CYBOZU_TEST_EQUAL(v.count("in"), 0);
 	CYBOZU_TEST_EQUAL(v.count("grand"), 1);
 
 	CYBOZU_TEST_EQUAL(v.count("the/at"), 0);
@@ -146,7 +146,7 @@ void printgamma(const cybozu::ldacvb0::Mat& gamma) {
 
 CYBOZU_TEST_AUTO(test_lda_cvb0_initialization)
 {
-	Documents<std::string, char> docs;
+	Documents<std::string, char> docs(cybozu::ldacvb0::REXWORD, true);
 	docs.vocabularies.add("a");	// 0
 	docs.vocabularies.add("b");	// 1
 	docs.vocabularies.add("c");	// 2