| // Copyright 2010-2015, Google Inc. |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: |
| // |
| // * Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above |
| // copyright notice, this list of conditions and the following disclaimer |
| // in the documentation and/or other materials provided with the |
| // distribution. |
| // * Neither the name of Google Inc. nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include "dictionary/user_dictionary.h" |
| |
| #include <algorithm> |
| #include <cstring> |
| #include <sstream> |
| #include <string> |
| #include <vector> |
| |
| #include "base/file_util.h" |
| #include "base/logging.h" |
| #include "base/number_util.h" |
| #include "base/port.h" |
| #include "base/singleton.h" |
| #include "base/system_util.h" |
| #include "base/trie.h" |
| #include "base/util.h" |
| #include "config/config.pb.h" |
| #include "config/config_handler.h" |
| #include "data_manager/testing/mock_user_pos_manager.h" |
| #include "dictionary/dictionary_test_util.h" |
| #include "dictionary/dictionary_token.h" |
| #include "dictionary/pos_matcher.h" |
| #include "dictionary/suppression_dictionary.h" |
| #include "dictionary/user_dictionary_storage.h" |
| #include "dictionary/user_pos.h" |
| #include "dictionary/user_pos_interface.h" |
| #include "testing/base/public/googletest.h" |
| #include "testing/base/public/gunit.h" |
| #include "usage_stats/usage_stats.h" |
| #include "usage_stats/usage_stats_testing_util.h" |
| |
| DECLARE_string(test_tmpdir); |
| |
| namespace mozc { |
| namespace dictionary { |
| namespace { |
| |
| const char kUserDictionary0[] = |
| "start\tstart\tverb\n" |
| "star\tstar\tnoun\n" |
| "starting\tstarting\tnoun\n" |
| "stamp\tstamp\tnoun\n" |
| "stand\tstand\tverb\n" |
| "smile\tsmile\tverb\n" |
| "smog\tsmog\tnoun\n" |
| // invalid characters "水雲" in reading |
| "\xE6\xB0\xB4\xE9\x9B\xB2\tvalue\tnoun\n" |
| |
| // Empty key |
| "\tvalue\tnoun\n" |
| // Empty value |
| "start\t\tnoun\n" |
| // Invalid POS |
| "star\tvalue\tpos\n" |
| // Empty POS |
| "star\tvalue\t\n" |
| // Duplicate entry |
| "start\tstart\tverb\n" |
| // The following are for tests for LookupComment |
| // No comment |
| "comment_key1\tcomment_value1\tnoun\n" |
| // Has comment |
| "comment_key2\tcomment_value2\tnoun\tcomment\n" |
| // Different POS |
| "comment_key3\tcomment_value3\tnoun\tcomment1\n" |
| "comment_key3\tcomment_value3\tverb\tcomment2\n" |
| // White spaces comment |
| "comment_key4\tcomment_value4\tverb\t \n"; |
| |
| const char kUserDictionary1[] = "end\tend\tverb\n"; |
| |
| void PushBackToken(const string &key, |
| const string &value, |
| uint16 id, |
| vector<UserPOS::Token> *tokens) { |
| tokens->resize(tokens->size() + 1); |
| UserPOS::Token *t = &tokens->back(); |
| t->key = key; |
| t->value = value; |
| t->id = id; |
| t->cost = 0; |
| } |
| |
| // This class is a mock class for writing unit tests of a class that |
| // depends on POS. It accepts only two values for part-of-speech: |
| // "noun" as words without inflection and "verb" as words with |
| // inflection. |
| class UserPOSMock : public UserPOSInterface { |
| public: |
| UserPOSMock() {} |
| virtual ~UserPOSMock() {} |
| |
| // This method returns true if the given pos is "noun" or "verb". |
| virtual bool IsValidPOS(const string &pos) const { |
| return true; |
| } |
| |
| static const char *kNoun; |
| static const char *kVerb; |
| |
| // Given a verb, this method expands it to three different forms, |
| // i.e. base form (the word itself), "-ed" form and "-ing" form. For |
| // example, if the given word is "play", the method returns "play", |
| // "played" and "playing". When a noun is passed, it returns only |
| // base form. The method set lid and rid of the word as following: |
| // |
| // POS | lid | rid |
| // ------------------+-----+----- |
| // noun | 100 | 100 |
| // verb (base form) | 200 | 200 |
| // verb (-ed form) | 210 | 210 |
| // verb (-ing form) | 220 | 220 |
| virtual bool GetTokens(const string &key, |
| const string &value, |
| const string &pos, |
| vector<UserPOS::Token> *tokens) const { |
| if (key.empty() || |
| value.empty() || |
| pos.empty() || |
| tokens == NULL) { |
| return false; |
| } |
| |
| tokens->clear(); |
| if (pos == kNoun) { |
| PushBackToken(key, value, 100, tokens); |
| return true; |
| } else if (pos == kVerb) { |
| PushBackToken(key, value, 200, tokens); |
| PushBackToken(key + "ed", value + "ed", 210, tokens); |
| PushBackToken(key + "ing", value + "ing", 220, tokens); |
| return true; |
| } else { |
| return false; |
| } |
| } |
| |
| virtual void GetPOSList(vector<string> *pos_list) const {} |
| |
| virtual bool GetPOSIDs(const string &pos, uint16 *id) const { |
| return false; |
| } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(UserPOSMock); |
| }; |
| // "名詞" |
| const char *UserPOSMock::kNoun = "\xE5\x90\x8D\xE8\xA9\x9E"; |
| // "動詞ワ行五段" |
| const char *UserPOSMock::kVerb = |
| "\xE5\x8B\x95\xE8\xA9\x9E\xE3\x83\xAF\xE8\xA1\x8C\xE4\xBA\x94\xE6\xAE\xB5"; |
| |
| string GenRandomAlphabet(int size) { |
| string result; |
| const size_t len = Util::Random(size) + 1; |
| for (int i = 0; i < len; ++i) { |
| const char32 l = Util::Random(static_cast<int>('z' - 'a')) + 'a'; |
| Util::UCS4ToUTF8Append(l, &result); |
| } |
| return result; |
| } |
| |
| class UserDictionaryTest : public ::testing::Test { |
| protected: |
| virtual void SetUp() { |
| SystemUtil::SetUserProfileDirectory(FLAGS_test_tmpdir); |
| suppression_dictionary_.reset(new SuppressionDictionary); |
| |
| mozc::usage_stats::UsageStats::ClearAllStatsForTest(); |
| } |
| |
| virtual void TearDown() { |
| mozc::usage_stats::UsageStats::ClearAllStatsForTest(); |
| } |
| |
| // Workaround for the constructor of UserDictionary being protected. |
| // Creates a user dictionary with mock pos data. |
| UserDictionary *CreateDictionaryWithMockPos() { |
| const testing::MockUserPosManager user_pos_manager; |
| return new UserDictionary( |
| new UserPOSMock(), |
| user_pos_manager.GetPOSMatcher(), |
| suppression_dictionary_.get()); |
| } |
| |
| // Creates a user dictionary with actual pos data. |
| UserDictionary *CreateDictionary() { |
| const testing::MockUserPosManager user_pos_manager; |
| return new UserDictionary(new UserPOS(user_pos_manager.GetUserPOSData()), |
| user_pos_manager.GetPOSMatcher(), |
| Singleton<SuppressionDictionary>::get()); |
| } |
| |
| struct Entry { |
| string key; |
| string value; |
| uint16 lid; |
| uint16 rid; |
| }; |
| |
| class EntryCollector : public DictionaryInterface::Callback { |
| public: |
| virtual ResultType OnToken(StringPiece, // key |
| StringPiece, // actual_key |
| const Token &token) { |
| // Collect only user dictionary entries. |
| if (token.attributes & Token::USER_DICTIONARY) { |
| entries_.push_back(Entry()); |
| entries_.back().key = token.key; |
| entries_.back().value = token.value; |
| entries_.back().lid = token.lid; |
| entries_.back().rid = token.rid; |
| } |
| return TRAVERSE_CONTINUE; |
| } |
| |
| const vector<Entry> &entries() const { return entries_; } |
| |
| private: |
| vector<Entry> entries_; |
| }; |
| |
| static void TestLookupPredictiveHelper(const Entry *expected, |
| size_t expected_size, |
| StringPiece key, |
| const UserDictionary &dic) { |
| EntryCollector collector; |
| dic.LookupPredictive(key, false, &collector); |
| |
| if (expected == NULL || expected_size == 0) { |
| EXPECT_TRUE(collector.entries().empty()); |
| } else { |
| ASSERT_FALSE(collector.entries().empty()); |
| CompareEntries(expected, expected_size, collector.entries()); |
| } |
| } |
| |
| static void TestLookupPrefixHelper(const Entry *expected, |
| size_t expected_size, |
| const char *key, |
| size_t key_size, |
| const UserDictionary &dic) { |
| EntryCollector collector; |
| dic.LookupPrefix(StringPiece(key, key_size), false, &collector); |
| |
| if (expected == NULL || expected_size == 0) { |
| EXPECT_TRUE(collector.entries().empty()); |
| } else { |
| ASSERT_FALSE(collector.entries().empty()); |
| CompareEntries(expected, expected_size, collector.entries()); |
| } |
| } |
| |
| static void TestLookupExactHelper(const Entry *expected, |
| size_t expected_size, |
| const char *key, |
| size_t key_size, |
| const UserDictionary &dic) { |
| EntryCollector collector; |
| dic.LookupExact(StringPiece(key, key_size), &collector); |
| |
| if (expected == NULL || expected_size == 0) { |
| EXPECT_TRUE(collector.entries().empty()); |
| } else { |
| ASSERT_FALSE(collector.entries().empty()); |
| CompareEntries(expected, expected_size, collector.entries()); |
| } |
| } |
| |
| static string EncodeEntry(const Entry &entry) { |
| return entry.key + "\t" + |
| entry.value + "\t" + |
| NumberUtil::SimpleItoa(entry.lid) + "\t" + |
| NumberUtil::SimpleItoa(entry.rid) + "\n"; |
| } |
| |
| static string EncodeEntries(const Entry *array, size_t size) { |
| vector<string> encoded_items; |
| for (size_t i = 0; i < size; ++i) { |
| encoded_items.push_back(EncodeEntry(array[i])); |
| } |
| sort(encoded_items.begin(), encoded_items.end()); |
| string result; |
| Util::JoinStrings(encoded_items, "", &result); |
| return result; |
| } |
| |
| static void CompareEntries(const Entry *expected, size_t expected_size, |
| const vector<Entry> &actual) { |
| const string expected_encoded = EncodeEntries(expected, expected_size); |
| const string actual_encoded = EncodeEntries(&actual[0], actual.size()); |
| EXPECT_EQ(expected_encoded, actual_encoded); |
| } |
| |
| static void LoadFromString(const string &contents, |
| UserDictionaryStorage *storage) { |
| istringstream is(contents); |
| CHECK(is.good()); |
| |
| storage->Clear(); |
| UserDictionaryStorage::UserDictionary *dic |
| = storage->add_dictionaries(); |
| CHECK(dic); |
| |
| string line; |
| while (!getline(is, line).fail()) { |
| if (line.empty() || line[0] == '#') { |
| continue; |
| } |
| vector<string> fields; |
| Util::SplitStringAllowEmpty(line, "\t", &fields); |
| EXPECT_GE(fields.size(), 3) << line; |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| CHECK(entry); |
| entry->set_key(fields[0]); |
| entry->set_value(fields[1]); |
| if (fields[2] == "verb") { |
| entry->set_pos(user_dictionary::UserDictionary::WA_GROUP1_VERB); |
| } else if (fields[2] == "noun") { |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| if (fields.size() >= 4 && !fields[3].empty()) { |
| entry->set_comment(fields[3]); |
| } |
| } |
| } |
| |
| // Helper function to lookup comment string from |dic|. |
| static string LookupComment(const UserDictionary& dic, |
| StringPiece key, StringPiece value) { |
| string comment; |
| dic.LookupComment(key, value, &comment); |
| return comment; |
| } |
| |
| scoped_ptr<SuppressionDictionary> suppression_dictionary_; |
| |
| private: |
| mozc::usage_stats::scoped_usage_stats_enabler usage_stats_enabler_; |
| }; |
| |
| TEST_F(UserDictionaryTest, TestLookupPredictive) { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| |
| { |
| UserDictionaryStorage storage(""); |
| UserDictionaryTest::LoadFromString(kUserDictionary0, &storage); |
| dic->Load(storage); |
| } |
| |
| // A normal lookup operation. |
| const Entry kExpected0[] = { |
| { "start", "start", 200, 200 }, |
| { "started", "started", 210, 210 }, |
| { "starting", "starting", 100, 100 }, |
| { "starting", "starting", 220, 220 }, |
| }; |
| TestLookupPredictiveHelper(kExpected0, arraysize(kExpected0), |
| "start", *dic.get()); |
| |
| // Another normal lookup operation. |
| const Entry kExpected1[] = { |
| { "stamp", "stamp", 100, 100 }, |
| { "stand", "stand", 200, 200 }, |
| { "standed", "standed", 210, 210 }, |
| { "standing", "standing", 220, 220 }, |
| { "star", "star", 100, 100 }, |
| { "start", "start", 200, 200 }, |
| { "started", "started", 210, 210 }, |
| { "starting", "starting", 100, 100 }, |
| { "starting", "starting", 220, 220 }, |
| }; |
| TestLookupPredictiveHelper(kExpected1, arraysize(kExpected1), |
| "st", *dic.get()); |
| |
| // Invalid input values should be just ignored. |
| TestLookupPredictiveHelper(NULL, 0, "", *dic.get()); |
| TestLookupPredictiveHelper(NULL, 0, |
| "\xE6\xB0\xB4\xE9\x9B\xB2", // "水雲" |
| *dic.get()); |
| |
| // Make a change to the dictionary file and load it again. |
| { |
| UserDictionaryStorage storage(""); |
| LoadFromString(kUserDictionary1, &storage); |
| dic->Load(storage); |
| } |
| |
| // A normal lookup again. |
| const Entry kExpected2[] = { |
| { "end", "end", 200, 200 }, |
| { "ended", "ended", 210, 210 }, |
| { "ending", "ending", 220, 220 }, |
| }; |
| TestLookupPredictiveHelper(kExpected2, arraysize(kExpected2), |
| "end", *dic.get()); |
| |
| // Entries in the dictionary before reloading cannot be looked up. |
| TestLookupPredictiveHelper(NULL, 0, "start", *dic.get()); |
| TestLookupPredictiveHelper(NULL, 0, "st", *dic.get()); |
| } |
| |
| TEST_F(UserDictionaryTest, TestLookupPrefix) { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| |
| { |
| UserDictionaryStorage storage(""); |
| LoadFromString(kUserDictionary0, &storage); |
| dic->Load(storage); |
| } |
| |
| // A normal lookup operation. |
| const Entry kExpected0[] = { |
| { "star", "star", 100, 100 }, |
| { "start", "start", 200, 200 }, |
| { "started", "started", 210, 210 }, |
| }; |
| TestLookupPrefixHelper(kExpected0, arraysize(kExpected0), |
| "started", 7, *dic.get()); |
| |
| // Another normal lookup operation. |
| const Entry kExpected1[] = { |
| { "star", "star", 100, 100 }, |
| { "start", "start", 200, 200 }, |
| { "starting", "starting", 100, 100 }, |
| { "starting", "starting", 220, 220 }, |
| }; |
| TestLookupPrefixHelper(kExpected1, arraysize(kExpected1), |
| "starting", 8, *dic.get()); |
| |
| // Invalid input values should be just ignored. |
| TestLookupPrefixHelper(NULL, 0, "", 0, *dic.get()); |
| TestLookupPrefixHelper( |
| NULL, 0, "\xE6\xB0\xB4\xE9\x9B\xB2", // "水雲" |
| strlen("\xE6\xB0\xB4\xE9\x9B\xB2"), *dic.get()); |
| |
| // Make a change to the dictionary file and load it again. |
| { |
| UserDictionaryStorage storage(""); |
| LoadFromString(kUserDictionary1, &storage); |
| dic->Load(storage); |
| } |
| |
| // A normal lookup. |
| const Entry kExpected2[] = { |
| { "end", "end", 200, 200 }, |
| { "ending", "ending", 220, 220 }, |
| }; |
| TestLookupPrefixHelper(kExpected2, arraysize(kExpected2), |
| "ending", 6, *dic.get()); |
| |
| // Lookup for entries which are gone should returns empty result. |
| TestLookupPrefixHelper(NULL, 0, "started", 7, *dic.get()); |
| TestLookupPrefixHelper(NULL, 0, "starting", 8, *dic.get()); |
| } |
| |
| TEST_F(UserDictionaryTest, TestLookupExact) { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| |
| { |
| UserDictionaryStorage storage(""); |
| LoadFromString(kUserDictionary0, &storage); |
| dic->Load(storage); |
| } |
| |
| // A normal lookup operation. |
| const Entry kExpected0[] = { |
| { "start", "start", 200, 200 }, |
| }; |
| TestLookupExactHelper(kExpected0, arraysize(kExpected0), |
| "start", 5, *dic.get()); |
| |
| // Another normal lookup operation. |
| const Entry kExpected1[] = { |
| { "starting", "starting", 100, 100 }, |
| { "starting", "starting", 220, 220 }, |
| }; |
| TestLookupExactHelper(kExpected1, arraysize(kExpected1), |
| "starting", 8, *dic.get()); |
| |
| // Invalid input values should be just ignored. |
| TestLookupPrefixHelper(NULL, 0, "", 0, *dic.get()); |
| TestLookupPrefixHelper(NULL, 0, "\xE6\xB0\xB4\xE9\x9B\xB2", // "水雲" |
| strlen("\xE6\xB0\xB4\xE9\x9B\xB2"), *dic.get()); |
| } |
| |
| TEST_F(UserDictionaryTest, TestLookupExactWithSuggestionOnlyWords) { |
| scoped_ptr<UserDictionary> user_dic(CreateDictionary()); |
| user_dic->WaitForReloader(); |
| |
| // Create dictionary |
| const string filename = FileUtil::JoinPath(FLAGS_test_tmpdir, |
| "suggestion_only_test.db"); |
| FileUtil::Unlink(filename); |
| UserDictionaryStorage storage(filename); |
| { |
| uint64 id = 0; |
| EXPECT_TRUE(storage.CreateDictionary("test", &id)); |
| UserDictionaryStorage::UserDictionary *dic = |
| storage.mutable_dictionaries(0); |
| |
| // "名詞" |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key("key"); |
| entry->set_value("noun"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| |
| // "サジェストのみ" |
| entry = dic->add_entries(); |
| entry->set_key("key"); |
| entry->set_value("suggest_only"); |
| entry->set_pos(user_dictionary::UserDictionary::SUGGESTION_ONLY); |
| |
| user_dic->Load(storage); |
| } |
| |
| // "suggestion_only" should not be looked up. |
| const testing::MockUserPosManager user_pos_manager; |
| const uint16 kNounId = user_pos_manager.GetPOSMatcher()->GetGeneralNounId(); |
| const Entry kExpected1[] = {{"key", "noun", kNounId, kNounId}}; |
| TestLookupExactHelper(kExpected1, arraysize(kExpected1), |
| "key", 3, *user_dic.get()); |
| } |
| |
| TEST_F(UserDictionaryTest, IncognitoModeTest) { |
| config::Config config; |
| config::ConfigHandler::GetConfig(&config); |
| config.set_incognito_mode(true); |
| config::ConfigHandler::SetConfig(config); |
| |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| |
| { |
| UserDictionaryStorage storage(""); |
| UserDictionaryTest::LoadFromString(kUserDictionary0, &storage); |
| dic->Load(storage); |
| } |
| |
| TestLookupPrefixHelper(NULL, 0, "start", 4, *dic); |
| TestLookupPredictiveHelper(NULL, 0, "s", *dic); |
| |
| config.set_incognito_mode(false); |
| config::ConfigHandler::SetConfig(config); |
| |
| { |
| EntryCollector collector; |
| dic->LookupPrefix("start", false, &collector); |
| EXPECT_FALSE(collector.entries().empty()); |
| } |
| { |
| EntryCollector collector; |
| dic->LookupPredictive("s", false, &collector); |
| EXPECT_FALSE(collector.entries().empty()); |
| } |
| } |
| |
| TEST_F(UserDictionaryTest, AsyncLoadTest) { |
| const string filename = FileUtil::JoinPath(FLAGS_test_tmpdir, |
| "async_load_test.db"); |
| FileUtil::Unlink(filename); |
| |
| // Create dictionary |
| vector<string> keys; |
| { |
| UserDictionaryStorage storage(filename); |
| |
| EXPECT_FALSE(storage.Load()); |
| EXPECT_TRUE(storage.Lock()); |
| |
| uint64 id = 0; |
| EXPECT_TRUE(storage.CreateDictionary("test", &id)); |
| UserDictionaryStorage::UserDictionary *dic = |
| storage.mutable_dictionaries(0); |
| for (size_t j = 0; j < 10000; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key(GenRandomAlphabet(10)); |
| entry->set_value(GenRandomAlphabet(10)); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| entry->set_comment(GenRandomAlphabet(10)); |
| keys.push_back(entry->key()); |
| } |
| EXPECT_TRUE(storage.Save()); |
| EXPECT_TRUE(storage.UnLock()); |
| } |
| |
| { |
| scoped_ptr<UserDictionary> dic(CreateDictionary()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| dic->SetUserDictionaryName(filename); |
| |
| for (int i = 0; i < 32; ++i) { |
| random_shuffle(keys.begin(), keys.end()); |
| dic->Reload(); |
| for (int i = 0; i < 1000; ++i) { |
| CollectTokenCallback callback; |
| dic->LookupPrefix(keys[i], false, &callback); |
| } |
| } |
| dic->WaitForReloader(); |
| } |
| FileUtil::Unlink(filename); |
| } |
| |
| TEST_F(UserDictionaryTest, AddToAutoRegisteredDictionary) { |
| const string filename = FileUtil::JoinPath(FLAGS_test_tmpdir, |
| "add_to_auto_registered.db"); |
| FileUtil::Unlink(filename); |
| |
| // Create dictionary |
| { |
| UserDictionaryStorage storage(filename); |
| EXPECT_FALSE(storage.Load()); |
| EXPECT_TRUE(storage.Lock()); |
| EXPECT_TRUE(storage.Save()); |
| EXPECT_TRUE(storage.UnLock()); |
| } |
| |
| // Add entries. |
| { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| dic->WaitForReloader(); |
| dic->SetUserDictionaryName(filename); |
| for (int i = 0; i < 100; ++i) { |
| EXPECT_TRUE(dic->AddToAutoRegisteredDictionary( |
| "key" + NumberUtil::SimpleItoa(i), |
| "value" + NumberUtil::SimpleItoa(i), |
| user_dictionary::UserDictionary::NOUN)); |
| dic->WaitForReloader(); |
| } |
| } |
| |
| // Verify the contents. |
| { |
| UserDictionaryStorage storage(filename); |
| EXPECT_TRUE(storage.Load()); |
| int index = 0; |
| EXPECT_EQ(1, storage.dictionaries_size()); |
| EXPECT_EQ(100, storage.dictionaries(index).entries_size()); |
| for (int i = 0; i < 100; ++i) { |
| EXPECT_EQ("key" + NumberUtil::SimpleItoa(i), |
| storage.dictionaries(index).entries(i).key()); |
| EXPECT_EQ("value" + NumberUtil::SimpleItoa(i), |
| storage.dictionaries(index).entries(i).value()); |
| EXPECT_EQ(user_dictionary::UserDictionary::NOUN, |
| storage.dictionaries(index).entries(i).pos()); |
| } |
| } |
| |
| FileUtil::Unlink(filename); |
| |
| // Create dictionary |
| { |
| UserDictionaryStorage storage(filename); |
| EXPECT_FALSE(storage.Load()); |
| EXPECT_TRUE(storage.Lock()); |
| EXPECT_TRUE(storage.Save()); |
| EXPECT_TRUE(storage.UnLock()); |
| } |
| |
| // Add same entries. |
| { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| dic->WaitForReloader(); |
| dic->SetUserDictionaryName(filename); |
| EXPECT_TRUE(dic->AddToAutoRegisteredDictionary( |
| "key", "value", user_dictionary::UserDictionary::NOUN)); |
| dic->WaitForReloader(); |
| // Duplicated one is not registered. |
| EXPECT_FALSE(dic->AddToAutoRegisteredDictionary( |
| "key", "value", user_dictionary::UserDictionary::NOUN)); |
| dic->WaitForReloader(); |
| } |
| |
| // Verify the contents. |
| { |
| UserDictionaryStorage storage(filename); |
| EXPECT_TRUE(storage.Load()); |
| EXPECT_EQ(1, storage.dictionaries_size()); |
| EXPECT_EQ(1, storage.dictionaries(0).entries_size()); |
| EXPECT_EQ("key", storage.dictionaries(0).entries(0).key()); |
| EXPECT_EQ("value", storage.dictionaries(0).entries(0).value()); |
| EXPECT_EQ(user_dictionary::UserDictionary::NOUN, |
| storage.dictionaries(0).entries(0).pos()); |
| } |
| } |
| |
| TEST_F(UserDictionaryTest, TestSuppressionDictionary) { |
| scoped_ptr<UserDictionary> user_dic(CreateDictionaryWithMockPos()); |
| user_dic->WaitForReloader(); |
| |
| const string filename = FileUtil::JoinPath(FLAGS_test_tmpdir, |
| "suppression_test.db"); |
| FileUtil::Unlink(filename); |
| |
| UserDictionaryStorage storage(filename); |
| |
| // Create dictionary |
| { |
| uint64 id = 0; |
| EXPECT_TRUE(storage.CreateDictionary("test", &id)); |
| UserDictionaryStorage::UserDictionary *dic = |
| storage.mutable_dictionaries(0); |
| for (size_t j = 0; j < 10000; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key("no_suppress_key" + |
| NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_value("no_suppress_value" + |
| NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| |
| for (size_t j = 0; j < 10; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key( |
| "suppress_key" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_value( |
| "suppress_value" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| // entry->set_pos("抑制単語"); |
| entry->set_pos(user_dictionary::UserDictionary::SUPPRESSION_WORD); |
| } |
| |
| suppression_dictionary_->Lock(); |
| EXPECT_TRUE(suppression_dictionary_->IsLocked()); |
| user_dic->Load(storage); |
| EXPECT_FALSE(suppression_dictionary_->IsLocked()); |
| |
| for (size_t j = 0; j < 10; ++j) { |
| EXPECT_TRUE(suppression_dictionary_->SuppressEntry( |
| "suppress_key" + NumberUtil::SimpleItoa(static_cast<uint32>(j)), |
| "suppress_value" + NumberUtil::SimpleItoa(static_cast<uint32>(j)))); |
| } |
| } |
| |
| // Remove suppression entry |
| { |
| storage.Clear(); |
| uint64 id = 0; |
| EXPECT_TRUE(storage.CreateDictionary("test", &id)); |
| UserDictionaryStorage::UserDictionary *dic = |
| storage.mutable_dictionaries(0); |
| for (size_t j = 0; j < 10000; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key( |
| "no_suppress_key" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_value( |
| "no_suppress_value" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| |
| suppression_dictionary_->Lock(); |
| user_dic->Load(storage); |
| EXPECT_FALSE(suppression_dictionary_->IsLocked()); |
| |
| for (size_t j = 0; j < 10; ++j) { |
| EXPECT_FALSE(suppression_dictionary_->SuppressEntry( |
| "suppress_key" + NumberUtil::SimpleItoa(static_cast<uint32>(j)), |
| "suppress_value" + NumberUtil::SimpleItoa(static_cast<uint32>(j)))); |
| } |
| } |
| FileUtil::Unlink(filename); |
| } |
| |
| TEST_F(UserDictionaryTest, TestSuggestionOnlyWord) { |
| scoped_ptr<UserDictionary> user_dic(CreateDictionary()); |
| user_dic->WaitForReloader(); |
| |
| const string filename = FileUtil::JoinPath(FLAGS_test_tmpdir, |
| "suggestion_only_test.db"); |
| FileUtil::Unlink(filename); |
| |
| UserDictionaryStorage storage(filename); |
| |
| // Create dictionary |
| { |
| uint64 id = 0; |
| EXPECT_TRUE(storage.CreateDictionary("test", &id)); |
| UserDictionaryStorage::UserDictionary *dic = |
| storage.mutable_dictionaries(0); |
| |
| for (size_t j = 0; j < 10; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key("key" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_value("default"); |
| // "名詞" |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| |
| for (size_t j = 0; j < 10; ++j) { |
| UserDictionaryStorage::UserDictionaryEntry *entry = |
| dic->add_entries(); |
| entry->set_key("key" + NumberUtil::SimpleItoa(static_cast<uint32>(j))); |
| entry->set_value("suggest_only"); |
| // "サジェストのみ" |
| entry->set_pos(user_dictionary::UserDictionary::SUGGESTION_ONLY); |
| } |
| |
| user_dic->Load(storage); |
| } |
| |
| { |
| const char kKey[] = "key0123"; |
| CollectTokenCallback callback; |
| user_dic->LookupPrefix(kKey, false, &callback); |
| const vector<Token> &tokens = callback.tokens(); |
| for (size_t i = 0; i < tokens.size(); ++i) { |
| EXPECT_EQ("default", tokens[i].value); |
| } |
| } |
| { |
| const char kKey[] = "key"; |
| CollectTokenCallback callback; |
| user_dic->LookupPredictive(kKey, false, &callback); |
| const vector<Token> &tokens = callback.tokens(); |
| for (size_t i = 0; i < tokens.size(); ++i) { |
| EXPECT_TRUE(tokens[i].value == "suggest_only" || |
| tokens[i].value == "default"); |
| } |
| } |
| |
| FileUtil::Unlink(filename); |
| } |
| |
| TEST_F(UserDictionaryTest, TestUsageStats) { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| UserDictionaryStorage storage(""); |
| |
| { |
| UserDictionaryStorage::UserDictionary *dic1 = storage.add_dictionaries(); |
| CHECK(dic1); |
| UserDictionaryStorage::UserDictionaryEntry *entry; |
| entry = dic1->add_entries(); |
| CHECK(entry); |
| entry->set_key("key1"); |
| entry->set_value("value1"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| entry = dic1->add_entries(); |
| CHECK(entry); |
| entry->set_key("key2"); |
| entry->set_value("value2"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| { |
| UserDictionaryStorage::UserDictionary *dic2 = storage.add_dictionaries(); |
| CHECK(dic2); |
| UserDictionaryStorage::UserDictionaryEntry *entry; |
| entry = dic2->add_entries(); |
| CHECK(entry); |
| entry->set_key("key3"); |
| entry->set_value("value3"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| entry = dic2->add_entries(); |
| CHECK(entry); |
| entry->set_key("key4"); |
| entry->set_value("value4"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| entry = dic2->add_entries(); |
| CHECK(entry); |
| entry->set_key("key5"); |
| entry->set_value("value5"); |
| entry->set_pos(user_dictionary::UserDictionary::NOUN); |
| } |
| dic->Load(storage); |
| |
| EXPECT_INTEGER_STATS("UserRegisteredWord", 5); |
| } |
| |
| TEST_F(UserDictionaryTest, LookupComment) { |
| scoped_ptr<UserDictionary> dic(CreateDictionaryWithMockPos()); |
| // Wait for async reload called from the constructor. |
| dic->WaitForReloader(); |
| |
| { |
| UserDictionaryStorage storage(""); |
| UserDictionaryTest::LoadFromString(kUserDictionary0, &storage); |
| dic->Load(storage); |
| } |
| |
| // Entry is in user dictionary but has no comment. |
| string comment; |
| comment = "prev comment"; |
| EXPECT_FALSE(dic->LookupComment("comment_key1", "comment_value2", &comment)); |
| EXPECT_EQ("prev comment", comment); |
| |
| // Usual case: single key-value pair with comment. |
| EXPECT_TRUE(dic->LookupComment("comment_key2", "comment_value2", &comment)); |
| EXPECT_EQ("comment", comment); |
| |
| // There exist two entries having the same key, value and POS. Since POS is |
| // irrelevant to comment lookup, the first nonempty comment should be found. |
| EXPECT_TRUE(dic->LookupComment("comment_key3", "comment_value3", &comment)); |
| EXPECT_EQ("comment1", comment); |
| |
| // White-space only comments should be cleared. |
| EXPECT_FALSE(dic->LookupComment("comment_key4", "comment_value4", &comment)); |
| // The previous comment should remain. |
| EXPECT_EQ("comment1", comment); |
| |
| // Comment should be found iff key and value match. |
| EXPECT_TRUE(LookupComment(*dic, "comment_key", "mismatching_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "comment_key1", "mismatching_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "comment_key2", "mismatching_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "comment_key3", "mismatching_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "comment_key4", "mismatching_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "mismatching_key", "comment_value").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "mismatching_key", "comment_value1").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "mismatching_key", "comment_value2").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "mismatching_key", "comment_value3").empty()); |
| EXPECT_TRUE(LookupComment(*dic, "mismatching_key", "comment_value4").empty()); |
| } |
| |
| } // namespace |
| } // namespace dictionary |
| } // namespace mozc |
