Remove FilterInfo from SystemDictionary
RegisterTokens and IsBadToken are inlined in RegisterReverseLookupResults to simplify. Also, TokenAfterSpellningToken is removed as spelling correction token is tested in LookupReverse test.
Note that this is just a code cleanup. No user-visible behavior change is intended in production code.
BUG=none
TEST=unittest
git-svn-id: https://mozc.googlecode.com/svn/trunk@544 a6090854-d499-a067-5803-1114d4e51264
diff --git a/src/dictionary/system/system_dictionary.cc b/src/dictionary/system/system_dictionary.cc
index e778415..17dffdd 100644
--- a/src/dictionary/system/system_dictionary.cc
+++ b/src/dictionary/system/system_dictionary.cc
@@ -1300,11 +1300,7 @@
for (set<int>::const_iterator set_itr = id_set.begin();
set_itr != id_set.end();
++set_itr) {
- FilterInfo filter;
- filter.conditions =
- (FilterInfo::VALUE_ID | FilterInfo::NO_SPELLING_CORRECTION);
- filter.value_id = *set_itr;
-
+ const int value_id = *set_itr;
typedef multimap<int, ReverseLookupResult>::const_iterator ResultItr;
pair<ResultItr, ResultItr> range = reverse_results.equal_range(*set_itr);
for (ResultItr result_itr = range.first;
@@ -1316,64 +1312,23 @@
key_trie_.RestoreKeyString(reverse_result.id_in_key_trie, buffer);
string tokens_key;
codec_->DecodeKey(encoded_key, &tokens_key);
- if (callback->OnKey(tokens_key) !=
- SystemDictionary::Callback::TRAVERSE_CONTINUE) {
+ if (callback->OnKey(tokens_key) != Callback::TRAVERSE_CONTINUE) {
continue;
}
-
- // actual_key is always the same as tokens_key for reverse conversions.
- RegisterTokens(
- filter,
- tokens_key,
- tokens_key,
- encoded_tokens_ptr + reverse_result.tokens_offset,
- callback);
+ for (TokenDecodeIterator iter(
+ codec_, value_trie_, frequent_pos_, tokens_key,
+ encoded_tokens_ptr + reverse_result.tokens_offset);
+ !iter.Done(); iter.Next()) {
+ const TokenInfo &token_info = iter.Get();
+ if (token_info.token->attributes & Token::SPELLING_CORRECTION ||
+ token_info.id_in_value_trie != value_id) {
+ continue;
+ }
+ callback->OnToken(tokens_key, tokens_key, *token_info.token);
+ }
}
}
}
-void SystemDictionary::RegisterTokens(
- const FilterInfo &filter,
- const string &tokens_key,
- const string &actual_key,
- const uint8 *encoded_tokens_ptr,
- Callback *callback) const {
- for (TokenDecodeIterator iter(codec_, value_trie_, frequent_pos_,
- actual_key, encoded_tokens_ptr);
- !iter.Done(); iter.Next()) {
- const TokenInfo &token_info = iter.Get();
- if (IsBadToken(filter, token_info)) {
- continue;
- }
- callback->OnToken(tokens_key, actual_key, *token_info.token);
- }
-}
-
-bool SystemDictionary::IsBadToken(
- const FilterInfo &filter,
- const TokenInfo &token_info) const {
- if ((filter.conditions & FilterInfo::NO_SPELLING_CORRECTION) &&
- (token_info.token->attributes & Token::SPELLING_CORRECTION)) {
- return true;
- }
-
- if ((filter.conditions & FilterInfo::VALUE_ID) &&
- token_info.id_in_value_trie != filter.value_id) {
- return true;
- }
-
- if ((filter.conditions & FilterInfo::ONLY_T13N) &&
- (token_info.value_type != TokenInfo::AS_IS_HIRAGANA &&
- token_info.value_type != TokenInfo::AS_IS_KATAKANA)) {
- // SAME_AS_PREV_VALUE may be t13n token.
- string hiragana;
- Util::KatakanaToHiragana(token_info.token->value, &hiragana);
- if (token_info.token->key != hiragana) {
- return true;
- }
- }
- return false;
-}
-
} // namespace dictionary
} // namespace mozc
diff --git a/src/dictionary/system/system_dictionary.h b/src/dictionary/system/system_dictionary.h
index ae571a1..d3266ec 100644
--- a/src/dictionary/system/system_dictionary.h
+++ b/src/dictionary/system/system_dictionary.h
@@ -46,13 +46,10 @@
#include "dictionary/system/words_info.h"
#include "storage/louds/bit_vector_based_array.h"
#include "storage/louds/louds_trie.h"
-// for FRIEND_TEST
-#include "testing/base/public/gunit_prod.h"
namespace mozc {
class DictionaryFile;
-struct Token;
namespace dictionary {
@@ -115,16 +112,6 @@
DISALLOW_COPY_AND_ASSIGN(Builder);
};
- struct ReverseLookupResult {
- ReverseLookupResult() : tokens_offset(-1), id_in_key_trie(-1) {}
- // Offset from the tokens section beginning.
- // (token_array_->Get(id_in_key_trie) ==
- // token_array_->Get(0) + tokens_offset)
- int tokens_offset;
- // Id in key trie
- int id_in_key_trie;
- };
-
virtual ~SystemDictionary();
// TODO(team): Use builder instead of following static methods.
@@ -157,56 +144,32 @@
virtual void ClearReverseLookupCache(
NodeAllocatorInterface *allocator) const;
- private:
- FRIEND_TEST(SystemDictionaryTest, TokenAfterSpellningToken);
-
- struct FilterInfo {
- enum Condition {
- NONE = 0,
- VALUE_ID = 1,
- NO_SPELLING_CORRECTION = 2,
- ONLY_T13N = 4,
- };
- int conditions;
- // Return results only for tokens with given |value_id|.
- // If VALUE_ID is specified
- int value_id;
- FilterInfo() : conditions(NONE), value_id(-1) {}
+ // TODO(noriyukit): This structure is implementation detail so should be
+ // hidden.
+ struct ReverseLookupResult {
+ ReverseLookupResult() : tokens_offset(-1), id_in_key_trie(-1) {}
+ // Offset from the tokens section beginning.
+ // (token_array_.Get(id_in_key_trie) == token_array_.Get(0) + tokens_offset)
+ int tokens_offset;
+ // Id in key trie
+ int id_in_key_trie;
};
+ private:
explicit SystemDictionary(const SystemDictionaryCodecInterface *codec);
-
bool OpenDictionaryFile(bool enable_reverse_lookup_index);
- // Calls |callback| with token info, which is filled using |tokens_key|,
- // |actual_key| and |encoded_tokens_ptr|.
- // |tokens_key| is a key used for look up.
- // |actual_key| is a token's key.
- // They may be different when we perform ambiguous search.
- void RegisterTokens(
- const FilterInfo &filter,
- const string &tokens_key,
- const string &actual_key,
- const uint8 *encoded_tokens_ptr,
- Callback *callback) const;
-
- bool IsBadToken(const FilterInfo &filter, const TokenInfo &token_info) const;
-
void RegisterReverseLookupTokensForT13N(StringPiece value,
Callback *callback) const;
-
void RegisterReverseLookupTokensForValue(StringPiece value,
NodeAllocatorInterface *allocator,
Callback *callback) const;
-
void ScanTokens(const set<int> &id_set,
multimap<int, ReverseLookupResult> *reverse_results) const;
-
void RegisterReverseLookupResults(
const set<int> &id_set,
const multimap<int, ReverseLookupResult> &reverse_results,
Callback *callback) const;
-
void InitReverseLookupIndex();
Callback::ResultType LookupPrefixWithKeyExpansionImpl(
diff --git a/src/dictionary/system/system_dictionary_test.cc b/src/dictionary/system/system_dictionary_test.cc
index 983b803..e01fbd4 100644
--- a/src/dictionary/system/system_dictionary_test.cc
+++ b/src/dictionary/system/system_dictionary_test.cc
@@ -1017,87 +1017,6 @@
}
}
-// Minimal modification of the codec for the TokenAfterSpellningToken.
-class CodecForTest : public SystemDictionaryCodecInterface {
- public:
- CodecForTest() : counter_(0) {
- }
-
- virtual ~CodecForTest() {
- }
-
- // Just mock methods.
- const string GetSectionNameForKey() const { return "Mock"; }
- const string GetSectionNameForValue() const { return "Mock"; }
- const string GetSectionNameForTokens() const { return "Mock"; }
- const string GetSectionNameForPos() const { return "Mock"; }
- void EncodeKey(const StringPiece src, string *dst) const {}
- void DecodeKey(const StringPiece src, string *dst) const {}
- size_t GetEncodedKeyLength(const StringPiece src) const { return 0; }
- size_t GetDecodedKeyLength(const StringPiece src) const { return 0; }
- void EncodeValue(const StringPiece src, string *dst) const {}
- void EncodeTokens(
- const vector<TokenInfo> &tokens, string *output) const {}
- void DecodeTokens(
- const uint8 *ptr, vector<TokenInfo> *tokens) const {}
- bool ReadTokenForReverseLookup(
- const uint8 *ptr, int *value_id, int *read_bytes) const { return false; }
- uint8 GetTokensTerminationFlag() const { return 0xff; }
-
- // Mock methods which will be actually called.
- bool DecodeToken(
- const uint8 *ptr, TokenInfo *token_info, int *read_bytes) const {
- *read_bytes = 0;
- switch (counter_++) {
- case 0:
- token_info->id_in_value_trie = 0;
- token_info->value_type = TokenInfo::DEFAULT_VALUE;
- token_info->token->attributes = Token::SPELLING_CORRECTION;
- token_info->token->cost = 1;
- return true;
- case 1:
- token_info->value_type = TokenInfo::SAME_AS_PREV_VALUE;
- token_info->token->cost = 111;
- return false;
- default:
- LOG(FATAL) << "Should never reach here.";
- }
- return true;
- }
-
- void DecodeValue(const StringPiece src, string *dst) const {
- *dst = "DummyValue";
- }
-
- private:
- mutable int counter_;
- DISALLOW_COPY_AND_ASSIGN(CodecForTest);
-};
-
-TEST_F(SystemDictionaryTest, TokenAfterSpellningToken) {
- scoped_ptr<SystemDictionary> system_dic(
- SystemDictionary::CreateSystemDictionaryFromFile(dic_fn_));
- // Filter for reverse look up.
- SystemDictionary::FilterInfo filter;
- filter.conditions = SystemDictionary::FilterInfo::NO_SPELLING_CORRECTION;
-
- // The 2nd token refers previous token by SAME_AS_PREV_VALUE,
- // but the 1st token is spelling correction which will be ignored for
- // reverse conversion.
-
- // Inject a minimal codec.
- scoped_ptr<CodecForTest> codec(new CodecForTest());
- system_dic->codec_ = codec.get();
-
- CollectTokenCallback callback;
- system_dic->RegisterTokens(filter, "", "", NULL, &callback);
-
- const vector<Token> &tokens = callback.tokens();
- EXPECT_EQ(1, tokens.size());
- EXPECT_EQ(111, tokens[0].cost);
- EXPECT_EQ("DummyValue", tokens[0].value);
-}
-
TEST_F(SystemDictionaryTest, EnableNoModifierTargetWithLoudsTrie) {
// "かつ"
const string k0 = "\xE3\x81\x8B\xE3\x81\xA4";
diff --git a/src/mozc_version_template.txt b/src/mozc_version_template.txt
index e67e6dc..de593f2 100644
--- a/src/mozc_version_template.txt
+++ b/src/mozc_version_template.txt
@@ -1,6 +1,6 @@
MAJOR=2
MINOR=16
-BUILD=2060
+BUILD=2061
REVISION=102
# NACL_DICTIONARY_VERSION is the target version of the system dictionary to be
# downloaded by NaCl Mozc.
