src/dictionary/user_dictionary.cc - mozc - Git at Google

 // Copyright 2010-2015, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
 // in the documentation and/or other materials provided with the
 // distribution.
 //     * Neither the name of Google Inc. nor the names of its
 // contributors may be used to endorse or promote products derived from
 // this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 #include "dictionary/user_dictionary.h"

 #include <algorithm>
 #include <limits>
 #include <set>
 #include <string>

 #include "base/compiler_specific.h"
 #include "base/logging.h"
 #include "base/mutex.h"
 #include "base/singleton.h"
 #include "base/stl_util.h"
 #include "base/thread.h"
 #include "base/util.h"
 #include "config/config.pb.h"
 #include "config/config_handler.h"
 #include "dictionary/dictionary_token.h"
 #include "dictionary/pos_matcher.h"
 #include "dictionary/suppression_dictionary.h"
 #include "dictionary/user_dictionary_storage.h"
 #include "dictionary/user_dictionary_util.h"
 #include "dictionary/user_pos.h"
 #include "usage_stats/usage_stats.h"

 namespace mozc {

 namespace {

 struct OrderByKey {
   bool operator()(const UserPOS::Token *lhs,
                   const UserPOS::Token *rhs) const {
     return lhs->key < rhs->key;
   }
 };

 struct OrderByKeyThenById {
   bool operator()(const UserPOS::Token *lhs,
                   const UserPOS::Token *rhs) const {
     const int comp = lhs->key.compare(rhs->key);
     return comp == 0 ? (lhs->id < rhs->id) : (comp < 0);
   }
 };

 class UserDictionaryFileManager {
  public:
   UserDictionaryFileManager() {}

   const string GetFileName() {
     scoped_lock l(&mutex_);
     if (filename_.empty()) {
       return UserDictionaryUtil::GetUserDictionaryFileName();
     } else {
       return filename_;
     }
   }

   void SetFileName(const string &filename) {
     scoped_lock l(&mutex_);
     filename_ = filename;
   }

  private:
   string filename_;
   Mutex mutex_;
   DISALLOW_COPY_AND_ASSIGN(UserDictionaryFileManager);
 };

 void FillTokenFromUserPOSToken(const UserPOS::Token &user_pos_token,
                                Token *token) {
   token->key = user_pos_token.key;
   token->value = user_pos_token.value;
   token->cost = user_pos_token.cost;
   token->lid = user_pos_token.id;
   token->rid = user_pos_token.id;
   token->attributes = Token::USER_DICTIONARY;
 }

 }  // namespace

 class TokensIndex : public vector<UserPOS::Token *> {
  public:
   explicit TokensIndex(const UserPOSInterface *user_pos,
                        SuppressionDictionary *suppression_dictionary)
       : user_pos_(user_pos),
         suppression_dictionary_(suppression_dictionary) {}
   virtual ~TokensIndex() {
     Clear();
   }

   void Clear() {
     STLDeleteElements(this);
     clear();
   }

   void Load(const user_dictionary::UserDictionaryStorage &storage) {
     Clear();
     set<uint64> seen;
     vector<UserPOS::Token> tokens;

     if (!suppression_dictionary_->IsLocked()) {
       LOG(ERROR) << "SuppressionDictionary must be locked first";
     }
     suppression_dictionary_->Clear();

     for (size_t i = 0; i < storage.dictionaries_size(); ++i) {
       const UserDictionaryStorage::UserDictionary &dic =
           storage.dictionaries(i);
       if (!dic.enabled() || dic.entries_size() == 0) {
         continue;
       }

       for (size_t j = 0; j < dic.entries_size(); ++j) {
         const UserDictionaryStorage::UserDictionaryEntry &entry =
             dic.entries(j);

         if (!UserDictionaryUtil::IsValidEntry(*user_pos_, entry)) {
           continue;
         }

         string tmp, reading;
         UserDictionaryUtil::NormalizeReading(entry.key(), &tmp);

         // We cannot call NormalizeVoiceSoundMark inside NormalizeReading,
         // because the normalization is user-visible.
         // http://b/2480844
         Util::NormalizeVoicedSoundMark(tmp, &reading);

         DCHECK_LE(0, entry.pos());
 MOZC_CLANG_PUSH_WARNING();
 #if MOZC_CLANG_HAS_WARNING(tautological-constant-out-of-range-compare)
 MOZC_CLANG_DISABLE_WARNING(tautological-constant-out-of-range-compare);
 #endif  // MOZC_CLANG_HAS_WARNING(tautological-constant-out-of-range-compare)
         DCHECK_LE(entry.pos(), 255);
 MOZC_CLANG_POP_WARNING();
         const uint64 fp = Util::Fingerprint(reading +
                                             "\t" +
                                             entry.value() +
                                             "\t" +
                                             static_cast<char>(entry.pos()));
         if (!seen.insert(fp).second) {
           VLOG(1) << "Found dup item";
           continue;
         }

         // "抑制単語"
         if (entry.pos() == user_dictionary::UserDictionary::SUPPRESSION_WORD) {
           suppression_dictionary_->AddEntry(reading, entry.value());
         } else {
           tokens.clear();
           user_pos_->GetTokens(
               reading, entry.value(),
               UserDictionaryUtil::GetStringPosType(entry.pos()), &tokens);
           for (size_t k = 0; k < tokens.size(); ++k) {
             this->push_back(new UserPOS::Token(tokens[k]));
             Util::StripWhiteSpaces(entry.comment(), &this->back()->comment);
           }
         }
       }
     }

     // Sort first by key and then by POS ID.
     sort(this->begin(), this->end(), OrderByKeyThenById());

     suppression_dictionary_->UnLock();

     VLOG(1) << this->size() << " user dic entries loaded";

     usage_stats::UsageStats::SetInteger("UserRegisteredWord",
                                         static_cast<int>(this->size()));
   }

  private:
   const UserPOSInterface *user_pos_;
   SuppressionDictionary *suppression_dictionary_;
 };

 class UserDictionaryReloader : public Thread {
  public:
   explicit UserDictionaryReloader(UserDictionary *dic)
       : auto_register_mode_(false), dic_(dic) {
     DCHECK(dic_);
   }

   virtual ~UserDictionaryReloader() {
     Join();
   }

   void StartAutoRegistration(const string &key,
                              const string &value,
                              user_dictionary::UserDictionary::PosType pos) {
     {
       scoped_lock l(&mutex_);
       auto_register_mode_ = true;
       key_ = key;
       value_ = value;
       pos_ = pos;
     }
     Start();
   }

   void StartReload() {
     Start();
   }

   virtual void Run() {
     scoped_ptr<UserDictionaryStorage> storage(new UserDictionaryStorage(
         Singleton<UserDictionaryFileManager>::get()->GetFileName()));

     // Load from file
     if (!storage->Load()) {
       return;
     }

     if (storage->ConvertSyncDictionariesToNormalDictionaries()) {
       LOG(INFO) << "Syncable dictionaries are converted to normal dictionaries";
       if (storage->Lock()) {
         storage->Save();
         storage->UnLock();
       }
     }

     if (auto_register_mode_ &&
         !storage->AddToAutoRegisteredDictionary(key_, value_, pos_)) {
       LOG(ERROR) << "failed to execute AddToAutoRegisteredDictionary";
       auto_register_mode_ = false;
       return;
     }

     auto_register_mode_ = false;
     dic_->Load(*(storage.get()));
   }

  private:
   Mutex mutex_;
   bool auto_register_mode_;
   UserDictionary *dic_;
   string key_;
   string value_;
   user_dictionary::UserDictionary::PosType pos_;

   DISALLOW_COPY_AND_ASSIGN(UserDictionaryReloader);
 };

 UserDictionary::UserDictionary(const UserPOSInterface *user_pos,
                                const POSMatcher *pos_matcher,
                                SuppressionDictionary *suppression_dictionary)
     : ALLOW_THIS_IN_INITIALIZER_LIST(
           reloader_(new UserDictionaryReloader(this))),
       user_pos_(user_pos),
       pos_matcher_(pos_matcher),
       suppression_dictionary_(suppression_dictionary),
       tokens_(new TokensIndex(user_pos_.get(), suppression_dictionary)),
       mutex_(new ReaderWriterMutex) {
   DCHECK(user_pos_.get());
   DCHECK(pos_matcher_);
   DCHECK(suppression_dictionary_);
   Reload();
 }

 UserDictionary::~UserDictionary() {
   reloader_->Join();
   delete tokens_;
 }

 bool UserDictionary::HasKey(StringPiece key) const {
   // TODO(noriyukit): Currently, we don't support HasKey() for user dictionary
   // because we need to search tokens linearly, which might be slow in extreme
   // cases where 100K entries exist.
   return false;
 }

 bool UserDictionary::HasValue(StringPiece value) const {
   // TODO(noriyukit): Currently, we don't support HasValue() for user dictionary
   // because we need to search tokens linearly, which might be slow in extreme
   // cases where 100K entries exist.  Note: HasValue() method is used only in
   // UserHistoryPredictor for privacy sensitivity check.
   return false;
 }

 void UserDictionary::LookupPredictive(
     StringPiece key,
     bool,  // use_kana_modifier_insensitive_lookup
     Callback *callback) const {
   scoped_reader_lock l(mutex_.get());

   if (key.empty()) {
     VLOG(2) << "string of length zero is passed.";
     return;
   }
   if (tokens_->empty()) {
     return;
   }
   if (GET_CONFIG(incognito_mode)) {
     return;
   }

   // Find the starting point of iteration over dictionary contents.
   UserPOS::Token key_token;
   key.CopyToString(&key_token.key);
   vector<UserPOS::Token *>::const_iterator it =
       lower_bound(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

   Token token;
   for (; it != tokens_->end(); ++it) {
     if (!Util::StartsWith((*it)->key, key)) {
       break;
     }
     switch (callback->OnKey((*it)->key)) {
       case Callback::TRAVERSE_DONE:
         return;
       case Callback::TRAVERSE_NEXT_KEY:
       case Callback::TRAVERSE_CULL:
         continue;
       default:
         break;
     }
     FillTokenFromUserPOSToken(**it, &token);
     // Override POS IDs for suggest only words.
     if (pos_matcher_->IsSuggestOnlyWord((*it)->id)) {
       token.lid = token.rid = pos_matcher_->GetUnknownId();
     }
     if (callback->OnToken((*it)->key, (*it)->key, token) ==
         Callback::TRAVERSE_DONE) {
       return;
     }
   }
 }

 // UserDictionary doesn't support kana modifier insensitive lookup.
 void UserDictionary::LookupPrefix(
     StringPiece key, bool /*use_kana_modifier_insensitive_lookup*/,
     Callback *callback) const {
   scoped_reader_lock l(mutex_.get());

   if (key.empty()) {
     LOG(WARNING) << "string of length zero is passed.";
     return;
   }
   if (tokens_->empty()) {
     return;
   }
   if (GET_CONFIG(incognito_mode)) {
     return;
   }

   // Find the starting point for iteration over dictionary contents.
   UserPOS::Token key_token;
   key_token.key.assign(key.data(), Util::OneCharLen(key.data()));
   vector<UserPOS::Token *>::const_iterator it =
       lower_bound(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

   Token token;
   for (; it != tokens_->end(); ++it) {
     if ((*it)->key > key) {
       break;
     }
     if (pos_matcher_->IsSuggestOnlyWord((*it)->id)) {
       continue;
     }
     if (!Util::StartsWith(key, (*it)->key)) {
       continue;
     }
     switch (callback->OnKey((*it)->key)) {
       case Callback::TRAVERSE_DONE:
         return;
       case Callback::TRAVERSE_NEXT_KEY:
         continue;
       case Callback::TRAVERSE_CULL:
         LOG(FATAL) << "UserDictionary doesn't support culling.";
         break;
       default:
         break;
     }
     FillTokenFromUserPOSToken(**it, &token);
     switch (callback->OnToken((*it)->key, (*it)->key, token)) {
       case Callback::TRAVERSE_DONE:
         return;
       case Callback::TRAVERSE_CULL:
         LOG(FATAL) << "UserDictionary doesn't support culling.";
         break;
       default:
         break;
     }
   }
 }

 void UserDictionary::LookupExact(StringPiece key, Callback *callback) const {
   scoped_reader_lock l(mutex_.get());
   if (key.empty() || tokens_->empty() || GET_CONFIG(incognito_mode)) {
     return;
   }
   UserPOS::Token key_token;
   key.CopyToString(&key_token.key);
   typedef vector<UserPOS::Token *>::const_iterator TokenIterator;
   pair<TokenIterator, TokenIterator> range =
       equal_range(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());
   if (range.first == range.second) {
     return;
   }
   if (callback->OnKey(key) != Callback::TRAVERSE_CONTINUE) {
     return;
   }

   Token token;
   for (; range.first != range.second; ++range.first) {
     const UserPOS::Token &user_pos_token = **range.first;
     if (pos_matcher_->IsSuggestOnlyWord(user_pos_token.id)) {
       continue;
     }
     FillTokenFromUserPOSToken(user_pos_token, &token);
     if (callback->OnToken(key, key, token) != Callback::TRAVERSE_CONTINUE) {
       return;
     }
   }
 }

 void UserDictionary::LookupReverse(StringPiece str,
                                    NodeAllocatorInterface *allocator,
                                    Callback *callback) const {
   if (GET_CONFIG(incognito_mode)) {
     return;
   }
 }

 bool UserDictionary::LookupComment(StringPiece key, StringPiece value,
                                    string *comment) const {
   if (key.empty() || GET_CONFIG(incognito_mode)) {
     return false;
   }

   scoped_reader_lock l(mutex_.get());
   if (tokens_->empty()) {
     return false;
   }

   UserPOS::Token key_token;
   key.CopyToString(&key_token.key);
   typedef vector<UserPOS::Token *>::const_iterator TokenIterator;
   pair<TokenIterator, TokenIterator> range =
       equal_range(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

   // Set the comment that was found first.
   for (; range.first != range.second; ++range.first) {
     const UserPOS::Token *token = *range.first;
     if (token->value == value && !token->comment.empty()) {
       comment->assign(token->comment);
       return true;
     }
   }
   return false;
 }

 bool UserDictionary::Reload() {
   if (reloader_->IsRunning()) {
     return false;
   }

   suppression_dictionary_->Lock();
   DCHECK(suppression_dictionary_->IsLocked());
   reloader_->StartReload();

   return true;
 }

 namespace {

 class FindValueCallback : public DictionaryInterface::Callback {
  public:
   explicit FindValueCallback(StringPiece value)
       : value_(value), found_(false) {}

   virtual ResultType OnToken(StringPiece,  // key
                              StringPiece,  // actual_key
                              const Token &token) {
     if (token.value == value_) {
       found_ = true;
       return TRAVERSE_DONE;
     }
     return TRAVERSE_CONTINUE;
   }

   bool found() const { return found_; }

  private:
   const StringPiece value_;
   bool found_;
 };

 }  // namespace

 bool UserDictionary::AddToAutoRegisteredDictionary(
     const string &key, const string &value,
     user_dictionary::UserDictionary::PosType pos) {
   if (reloader_->IsRunning()) {
     return false;
   }

   FindValueCallback callback(value);
   LookupExact(key, &callback);
   if (callback.found()) {
     // Already registered.
     return false;
   }

   suppression_dictionary_->Lock();
   DCHECK(suppression_dictionary_->IsLocked());
   reloader_->StartAutoRegistration(key, value, pos);

   return true;
 }

 void UserDictionary::WaitForReloader() {
   reloader_->Join();
 }

 void UserDictionary::Swap(TokensIndex *new_tokens) {
   DCHECK(new_tokens);
   TokensIndex *old_tokens = tokens_;
   {
     scoped_writer_lock l(mutex_.get());
     tokens_ = new_tokens;
   }
   delete old_tokens;
 }

 bool UserDictionary::Load(
     const user_dictionary::UserDictionaryStorage &storage) {
   size_t size = 0;
   {
     scoped_reader_lock l(mutex_.get());
     size = tokens_->size();
   }

   // If UserDictionary is pretty big, we first remove the
   // current dictionary to save memory usage.
 #ifdef OS_ANDROID
   const size_t kVeryBigUserDictionarySize = 5000;
 #else
   const size_t kVeryBigUserDictionarySize = 100000;
 #endif

   if (size >= kVeryBigUserDictionarySize) {
     TokensIndex *dummy_empty_tokens = new TokensIndex(user_pos_.get(),
                                                       suppression_dictionary_);
     Swap(dummy_empty_tokens);
   }

   TokensIndex *tokens = new TokensIndex(user_pos_.get(),
                                         suppression_dictionary_);
   tokens->Load(storage);
   Swap(tokens);
   return true;
 }

 void UserDictionary::SetUserDictionaryName(const string &filename) {
   Singleton<UserDictionaryFileManager>::get()->SetFileName(filename);
 }

 }  // namespace mozc
	// Copyright 2010-2015, Google Inc.
	// All rights reserved.
	//
	// Redistribution and use in source and binary forms, with or without
	// modification, are permitted provided that the following conditions are
	// met:
	//
	// * Redistributions of source code must retain the above copyright
	// notice, this list of conditions and the following disclaimer.
	// * Redistributions in binary form must reproduce the above
	// copyright notice, this list of conditions and the following disclaimer
	// in the documentation and/or other materials provided with the
	// distribution.
	// * Neither the name of Google Inc. nor the names of its
	// contributors may be used to endorse or promote products derived from
	// this software without specific prior written permission.
	//
	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	#include "dictionary/user_dictionary.h"

	#include <algorithm>
	#include <limits>
	#include <set>
	#include <string>

	#include "base/compiler_specific.h"
	#include "base/logging.h"
	#include "base/mutex.h"
	#include "base/singleton.h"
	#include "base/stl_util.h"
	#include "base/thread.h"
	#include "base/util.h"
	#include "config/config.pb.h"
	#include "config/config_handler.h"
	#include "dictionary/dictionary_token.h"
	#include "dictionary/pos_matcher.h"
	#include "dictionary/suppression_dictionary.h"
	#include "dictionary/user_dictionary_storage.h"
	#include "dictionary/user_dictionary_util.h"
	#include "dictionary/user_pos.h"
	#include "usage_stats/usage_stats.h"

	namespace mozc {

	namespace {

	struct OrderByKey {
	bool operator()(const UserPOS::Token *lhs,
	const UserPOS::Token *rhs) const {
	return lhs->key < rhs->key;
	}
	};

	struct OrderByKeyThenById {
	bool operator()(const UserPOS::Token *lhs,
	const UserPOS::Token *rhs) const {
	const int comp = lhs->key.compare(rhs->key);
	return comp == 0 ? (lhs->id < rhs->id) : (comp < 0);
	}
	};

	class UserDictionaryFileManager {
	public:
	UserDictionaryFileManager() {}

	const string GetFileName() {
	scoped_lock l(&mutex_);
	if (filename_.empty()) {
	return UserDictionaryUtil::GetUserDictionaryFileName();
	} else {
	return filename_;
	}
	}

	void SetFileName(const string &filename) {
	scoped_lock l(&mutex_);
	filename_ = filename;
	}

	private:
	string filename_;
	Mutex mutex_;
	DISALLOW_COPY_AND_ASSIGN(UserDictionaryFileManager);
	};

	void FillTokenFromUserPOSToken(const UserPOS::Token &user_pos_token,
	Token *token) {
	token->key = user_pos_token.key;
	token->value = user_pos_token.value;
	token->cost = user_pos_token.cost;
	token->lid = user_pos_token.id;
	token->rid = user_pos_token.id;
	token->attributes = Token::USER_DICTIONARY;
	}

	} // namespace

	class TokensIndex : public vector<UserPOS::Token *> {
	public:
	explicit TokensIndex(const UserPOSInterface *user_pos,
	SuppressionDictionary *suppression_dictionary)
	: user_pos_(user_pos),
	suppression_dictionary_(suppression_dictionary) {}
	virtual ~TokensIndex() {
	Clear();
	}

	void Clear() {
	STLDeleteElements(this);
	clear();
	}

	void Load(const user_dictionary::UserDictionaryStorage &storage) {
	Clear();
	set<uint64> seen;
	vector<UserPOS::Token> tokens;

	if (!suppression_dictionary_->IsLocked()) {
	LOG(ERROR) << "SuppressionDictionary must be locked first";
	}
	suppression_dictionary_->Clear();

	for (size_t i = 0; i < storage.dictionaries_size(); ++i) {
	const UserDictionaryStorage::UserDictionary &dic =
	storage.dictionaries(i);
	if (!dic.enabled() \|\| dic.entries_size() == 0) {
	continue;
	}

	for (size_t j = 0; j < dic.entries_size(); ++j) {
	const UserDictionaryStorage::UserDictionaryEntry &entry =
	dic.entries(j);

	if (!UserDictionaryUtil::IsValidEntry(*user_pos_, entry)) {
	continue;
	}

	string tmp, reading;
	UserDictionaryUtil::NormalizeReading(entry.key(), &tmp);

	// We cannot call NormalizeVoiceSoundMark inside NormalizeReading,
	// because the normalization is user-visible.
	// http://b/2480844
	Util::NormalizeVoicedSoundMark(tmp, &reading);

	DCHECK_LE(0, entry.pos());
	MOZC_CLANG_PUSH_WARNING();
	#if MOZC_CLANG_HAS_WARNING(tautological-constant-out-of-range-compare)
	MOZC_CLANG_DISABLE_WARNING(tautological-constant-out-of-range-compare);
	#endif // MOZC_CLANG_HAS_WARNING(tautological-constant-out-of-range-compare)
	DCHECK_LE(entry.pos(), 255);
	MOZC_CLANG_POP_WARNING();
	const uint64 fp = Util::Fingerprint(reading +
	"\t" +
	entry.value() +
	"\t" +
	static_cast<char>(entry.pos()));
	if (!seen.insert(fp).second) {
	VLOG(1) << "Found dup item";
	continue;
	}

	// "抑制単語"
	if (entry.pos() == user_dictionary::UserDictionary::SUPPRESSION_WORD) {
	suppression_dictionary_->AddEntry(reading, entry.value());
	} else {
	tokens.clear();
	user_pos_->GetTokens(
	reading, entry.value(),
	UserDictionaryUtil::GetStringPosType(entry.pos()), &tokens);
	for (size_t k = 0; k < tokens.size(); ++k) {
	this->push_back(new UserPOS::Token(tokens[k]));
	Util::StripWhiteSpaces(entry.comment(), &this->back()->comment);
	}
	}
	}
	}

	// Sort first by key and then by POS ID.
	sort(this->begin(), this->end(), OrderByKeyThenById());

	suppression_dictionary_->UnLock();

	VLOG(1) << this->size() << " user dic entries loaded";

	usage_stats::UsageStats::SetInteger("UserRegisteredWord",
	static_cast<int>(this->size()));
	}

	private:
	const UserPOSInterface *user_pos_;
	SuppressionDictionary *suppression_dictionary_;
	};

	class UserDictionaryReloader : public Thread {
	public:
	explicit UserDictionaryReloader(UserDictionary *dic)
	: auto_register_mode_(false), dic_(dic) {
	DCHECK(dic_);
	}

	virtual ~UserDictionaryReloader() {
	Join();
	}

	void StartAutoRegistration(const string &key,
	const string &value,
	user_dictionary::UserDictionary::PosType pos) {
	{
	scoped_lock l(&mutex_);
	auto_register_mode_ = true;
	key_ = key;
	value_ = value;
	pos_ = pos;
	}
	Start();
	}

	void StartReload() {
	Start();
	}

	virtual void Run() {
	scoped_ptr<UserDictionaryStorage> storage(new UserDictionaryStorage(
	Singleton<UserDictionaryFileManager>::get()->GetFileName()));

	// Load from file
	if (!storage->Load()) {
	return;
	}

	if (storage->ConvertSyncDictionariesToNormalDictionaries()) {
	LOG(INFO) << "Syncable dictionaries are converted to normal dictionaries";
	if (storage->Lock()) {
	storage->Save();
	storage->UnLock();
	}
	}

	if (auto_register_mode_ &&
	!storage->AddToAutoRegisteredDictionary(key_, value_, pos_)) {
	LOG(ERROR) << "failed to execute AddToAutoRegisteredDictionary";
	auto_register_mode_ = false;
	return;
	}

	auto_register_mode_ = false;
	dic_->Load(*(storage.get()));
	}

	private:
	Mutex mutex_;
	bool auto_register_mode_;
	UserDictionary *dic_;
	string key_;
	string value_;
	user_dictionary::UserDictionary::PosType pos_;

	DISALLOW_COPY_AND_ASSIGN(UserDictionaryReloader);
	};

	UserDictionary::UserDictionary(const UserPOSInterface *user_pos,
	const POSMatcher *pos_matcher,
	SuppressionDictionary *suppression_dictionary)
	: ALLOW_THIS_IN_INITIALIZER_LIST(
	reloader_(new UserDictionaryReloader(this))),
	user_pos_(user_pos),
	pos_matcher_(pos_matcher),
	suppression_dictionary_(suppression_dictionary),
	tokens_(new TokensIndex(user_pos_.get(), suppression_dictionary)),
	mutex_(new ReaderWriterMutex) {
	DCHECK(user_pos_.get());
	DCHECK(pos_matcher_);
	DCHECK(suppression_dictionary_);
	Reload();
	}

	UserDictionary::~UserDictionary() {
	reloader_->Join();
	delete tokens_;
	}

	bool UserDictionary::HasKey(StringPiece key) const {
	// TODO(noriyukit): Currently, we don't support HasKey() for user dictionary
	// because we need to search tokens linearly, which might be slow in extreme
	// cases where 100K entries exist.
	return false;
	}

	bool UserDictionary::HasValue(StringPiece value) const {
	// TODO(noriyukit): Currently, we don't support HasValue() for user dictionary
	// because we need to search tokens linearly, which might be slow in extreme
	// cases where 100K entries exist. Note: HasValue() method is used only in
	// UserHistoryPredictor for privacy sensitivity check.
	return false;
	}

	void UserDictionary::LookupPredictive(
	StringPiece key,
	bool, // use_kana_modifier_insensitive_lookup
	Callback *callback) const {
	scoped_reader_lock l(mutex_.get());

	if (key.empty()) {
	VLOG(2) << "string of length zero is passed.";
	return;
	}
	if (tokens_->empty()) {
	return;
	}
	if (GET_CONFIG(incognito_mode)) {
	return;
	}

	// Find the starting point of iteration over dictionary contents.
	UserPOS::Token key_token;
	key.CopyToString(&key_token.key);
	vector<UserPOS::Token *>::const_iterator it =
	lower_bound(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

	Token token;
	for (; it != tokens_->end(); ++it) {
	if (!Util::StartsWith((*it)->key, key)) {
	break;
	}
	switch (callback->OnKey((*it)->key)) {
	case Callback::TRAVERSE_DONE:
	return;
	case Callback::TRAVERSE_NEXT_KEY:
	case Callback::TRAVERSE_CULL:
	continue;
	default:
	break;
	}
	FillTokenFromUserPOSToken(**it, &token);
	// Override POS IDs for suggest only words.
	if (pos_matcher_->IsSuggestOnlyWord((*it)->id)) {
	token.lid = token.rid = pos_matcher_->GetUnknownId();
	}
	if (callback->OnToken((it)->key, (it)->key, token) ==
	Callback::TRAVERSE_DONE) {
	return;
	}
	}
	}

	// UserDictionary doesn't support kana modifier insensitive lookup.
	void UserDictionary::LookupPrefix(
	StringPiece key, bool /use_kana_modifier_insensitive_lookup/,
	Callback *callback) const {
	scoped_reader_lock l(mutex_.get());

	if (key.empty()) {
	LOG(WARNING) << "string of length zero is passed.";
	return;
	}
	if (tokens_->empty()) {
	return;
	}
	if (GET_CONFIG(incognito_mode)) {
	return;
	}

	// Find the starting point for iteration over dictionary contents.
	UserPOS::Token key_token;
	key_token.key.assign(key.data(), Util::OneCharLen(key.data()));
	vector<UserPOS::Token *>::const_iterator it =
	lower_bound(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

	Token token;
	for (; it != tokens_->end(); ++it) {
	if ((*it)->key > key) {
	break;
	}
	if (pos_matcher_->IsSuggestOnlyWord((*it)->id)) {
	continue;
	}
	if (!Util::StartsWith(key, (*it)->key)) {
	continue;
	}
	switch (callback->OnKey((*it)->key)) {
	case Callback::TRAVERSE_DONE:
	return;
	case Callback::TRAVERSE_NEXT_KEY:
	continue;
	case Callback::TRAVERSE_CULL:
	LOG(FATAL) << "UserDictionary doesn't support culling.";
	break;
	default:
	break;
	}
	FillTokenFromUserPOSToken(**it, &token);
	switch (callback->OnToken((it)->key, (it)->key, token)) {
	case Callback::TRAVERSE_DONE:
	return;
	case Callback::TRAVERSE_CULL:
	LOG(FATAL) << "UserDictionary doesn't support culling.";
	break;
	default:
	break;
	}
	}
	}

	void UserDictionary::LookupExact(StringPiece key, Callback *callback) const {
	scoped_reader_lock l(mutex_.get());
	if (key.empty() \|\| tokens_->empty() \|\| GET_CONFIG(incognito_mode)) {
	return;
	}
	UserPOS::Token key_token;
	key.CopyToString(&key_token.key);
	typedef vector<UserPOS::Token *>::const_iterator TokenIterator;
	pair<TokenIterator, TokenIterator> range =
	equal_range(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());
	if (range.first == range.second) {
	return;
	}
	if (callback->OnKey(key) != Callback::TRAVERSE_CONTINUE) {
	return;
	}

	Token token;
	for (; range.first != range.second; ++range.first) {
	const UserPOS::Token &user_pos_token = **range.first;
	if (pos_matcher_->IsSuggestOnlyWord(user_pos_token.id)) {
	continue;
	}
	FillTokenFromUserPOSToken(user_pos_token, &token);
	if (callback->OnToken(key, key, token) != Callback::TRAVERSE_CONTINUE) {
	return;
	}
	}
	}

	void UserDictionary::LookupReverse(StringPiece str,
	NodeAllocatorInterface *allocator,
	Callback *callback) const {
	if (GET_CONFIG(incognito_mode)) {
	return;
	}
	}

	bool UserDictionary::LookupComment(StringPiece key, StringPiece value,
	string *comment) const {
	if (key.empty() \|\| GET_CONFIG(incognito_mode)) {
	return false;
	}

	scoped_reader_lock l(mutex_.get());
	if (tokens_->empty()) {
	return false;
	}

	UserPOS::Token key_token;
	key.CopyToString(&key_token.key);
	typedef vector<UserPOS::Token *>::const_iterator TokenIterator;
	pair<TokenIterator, TokenIterator> range =
	equal_range(tokens_->begin(), tokens_->end(), &key_token, OrderByKey());

	// Set the comment that was found first.
	for (; range.first != range.second; ++range.first) {
	const UserPOS::Token token = range.first;
	if (token->value == value && !token->comment.empty()) {
	comment->assign(token->comment);
	return true;
	}
	}
	return false;
	}

	bool UserDictionary::Reload() {
	if (reloader_->IsRunning()) {
	return false;
	}

	suppression_dictionary_->Lock();
	DCHECK(suppression_dictionary_->IsLocked());
	reloader_->StartReload();

	return true;
	}

	namespace {

	class FindValueCallback : public DictionaryInterface::Callback {
	public:
	explicit FindValueCallback(StringPiece value)
	: value_(value), found_(false) {}

	virtual ResultType OnToken(StringPiece, // key
	StringPiece, // actual_key
	const Token &token) {
	if (token.value == value_) {
	found_ = true;
	return TRAVERSE_DONE;
	}
	return TRAVERSE_CONTINUE;
	}

	bool found() const { return found_; }

	private:
	const StringPiece value_;
	bool found_;
	};

	} // namespace

	bool UserDictionary::AddToAutoRegisteredDictionary(
	const string &key, const string &value,
	user_dictionary::UserDictionary::PosType pos) {
	if (reloader_->IsRunning()) {
	return false;
	}

	FindValueCallback callback(value);
	LookupExact(key, &callback);
	if (callback.found()) {
	// Already registered.
	return false;
	}

	suppression_dictionary_->Lock();
	DCHECK(suppression_dictionary_->IsLocked());
	reloader_->StartAutoRegistration(key, value, pos);

	return true;
	}

	void UserDictionary::WaitForReloader() {
	reloader_->Join();
	}

	void UserDictionary::Swap(TokensIndex *new_tokens) {
	DCHECK(new_tokens);
	TokensIndex *old_tokens = tokens_;
	{
	scoped_writer_lock l(mutex_.get());
	tokens_ = new_tokens;
	}
	delete old_tokens;
	}

	bool UserDictionary::Load(
	const user_dictionary::UserDictionaryStorage &storage) {
	size_t size = 0;
	{
	scoped_reader_lock l(mutex_.get());
	size = tokens_->size();
	}

	// If UserDictionary is pretty big, we first remove the
	// current dictionary to save memory usage.
	#ifdef OS_ANDROID
	const size_t kVeryBigUserDictionarySize = 5000;
	#else
	const size_t kVeryBigUserDictionarySize = 100000;
	#endif

	if (size >= kVeryBigUserDictionarySize) {
	TokensIndex *dummy_empty_tokens = new TokensIndex(user_pos_.get(),
	suppression_dictionary_);
	Swap(dummy_empty_tokens);
	}

	TokensIndex *tokens = new TokensIndex(user_pos_.get(),
	suppression_dictionary_);
	tokens->Load(storage);
	Swap(tokens);
	return true;
	}

	void UserDictionary::SetUserDictionaryName(const string &filename) {
	Singleton<UserDictionaryFileManager>::get()->SetFileName(filename);
	}

	} // namespace mozc