src/dictionary/system/value_dictionary.cc

// Copyright 2010-2015, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "dictionary/system/value_dictionary.h"

#include <limits>
#include <queue>
#include <string>

#include "base/logging.h"
#include "base/port.h"
#include "base/string_piece.h"
#include "base/system_util.h"
#include "dictionary/dictionary_token.h"
#include "dictionary/file/dictionary_file.h"
#include "dictionary/pos_matcher.h"
#include "dictionary/system/codec_interface.h"
#include "storage/louds/louds_trie.h"

using mozc::storage::louds::LoudsTrie;

namespace mozc {
namespace dictionary {

ValueDictionary::ValueDictionary(const POSMatcher& pos_matcher)
    : dictionary_file_(new DictionaryFile),
      codec_(SystemDictionaryCodecFactory::GetCodec()),
      suggestion_only_word_id_(pos_matcher.GetSuggestOnlyWordId()) {
}

ValueDictionary::~ValueDictionary() {}

// static
ValueDictionary *ValueDictionary::CreateValueDictionaryFromFile(
    const POSMatcher& pos_matcher, const string &filename) {
  scoped_ptr<ValueDictionary> instance(new ValueDictionary(pos_matcher));
  DCHECK(instance.get());
  if (!instance->dictionary_file_->OpenFromFile(filename)) {
    LOG(ERROR) << "Failed to open system dictionary file";
    return nullptr;
  }
  if (!instance->OpenDictionaryFile()) {
    LOG(ERROR) << "Failed to create value dictionary";
    return nullptr;
  }
  return instance.release();
}

// static
ValueDictionary *ValueDictionary::CreateValueDictionaryFromImage(
    const POSMatcher& pos_matcher, const char *ptr, int len) {
  // Make the dictionary not to be paged out.
  // We don't check the return value because the process doesn't necessarily
  // has the priviledge to mlock.
  // Note that we don't munlock the space because it's always better to keep
  // the singleton system dictionary paged in as long as the process runs.
  SystemUtil::MaybeMLock(ptr, len);
  scoped_ptr<ValueDictionary> instance(new ValueDictionary(pos_matcher));
  DCHECK(instance.get());
  if (!instance->dictionary_file_->OpenFromImage(ptr, len)) {
    LOG(ERROR) << "Failed to open system dictionary file";
    return nullptr;
  }
  if (!instance->OpenDictionaryFile()) {
    LOG(ERROR) << "Failed to create value dictionary";
    return nullptr;
  }
  return instance.release();
}

bool ValueDictionary::OpenDictionaryFile() {
  int image_len = 0;
  const unsigned char *value_image =
      reinterpret_cast<const uint8 *>(dictionary_file_->GetSection(
          codec_->GetSectionNameForValue(), &image_len));
  CHECK(value_image) << "can not find value section";
  if (!(value_trie_.Open(value_image))) {
    DLOG(ERROR) << "Cannot open value trie";
    return false;
  }
  return true;
}

// ValueDictionary is supposed to use the same data with SystemDictionary
// and SystemDictionary::HasKey should return the same result with
// ValueDictionary::HasKey.  So we can skip the actual logic of HasKey
// and return just false.
bool ValueDictionary::HasKey(StringPiece key) const {
  return false;
}

// ValueDictionary is supposed to use the same data with SystemDictionary
// and SystemDictionary::HasValue should return the same result with
// ValueDictionary::HasValue.  So we can skip the actual logic of HasValue
// and return just false.
bool ValueDictionary::HasValue(StringPiece value) const {
  return false;
}

namespace {

// A version of the above function for Token.
inline void FillToken(const uint16 suggestion_only_word_id,
                      StringPiece key, Token *token) {
  key.CopyToString(&token->key);
  token->value = token->key;
  token->cost = 10000;
  token->lid = token->rid = suggestion_only_word_id;
  token->attributes = Token::NONE;
}

inline DictionaryInterface::Callback::ResultType HandleTerminalNode(
    const LoudsTrie &value_trie,
    const SystemDictionaryCodecInterface &codec,
    const uint16 suggestion_only_word_id,
    const LoudsTrie::Node &node,
    DictionaryInterface::Callback *callback,
    char *encoded_value_buffer,
    string *value,
    Token *token) {
  const StringPiece encoded_value =
      value_trie.RestoreKeyString(node, encoded_value_buffer);

  value->clear();
  codec.DecodeValue(encoded_value, value);
  const DictionaryInterface::Callback::ResultType result =
      callback->OnKey(*value);
  if (result != DictionaryInterface::Callback::TRAVERSE_CONTINUE) {
    return result;
  }

  FillToken(suggestion_only_word_id, *value, token);
  return callback->OnToken(*value, *value, *token);
}

}  // namespace

void ValueDictionary::LookupPredictive(
    StringPiece key,
    bool,  // use_kana_modifier_insensitive_lookup
    Callback *callback) const {
  // Do nothing for empty key, although looking up all the entries with empty
  // string seems natural.
  if (key.empty()) {
    return;
  }
  string encoded_key;
  codec_->EncodeValue(key, &encoded_key);

  LoudsTrie::Node node;
  if (!value_trie_.Traverse(encoded_key, &node)) {
    return;
  }

  char encoded_value_buffer[LoudsTrie::kMaxDepth + 1];
  string value;
  value.reserve(key.size() * 2);
  Token token;

  // Traverse subtree rooted at |node|.
  queue<LoudsTrie::Node> queue;
  queue.push(node);
  do {
    node = queue.front();
    queue.pop();

    if (value_trie_.IsTerminalNode(node)) {
      switch (HandleTerminalNode(value_trie_, *codec_,
                                 suggestion_only_word_id_,
                                 node, callback, encoded_value_buffer,
                                 &value, &token)) {
        case Callback::TRAVERSE_DONE:
          return;
        case Callback::TRAVERSE_CULL:
          continue;
        default:
          break;
      }
    }

    for (value_trie_.MoveToFirstChild(&node);
         value_trie_.IsValidNode(node);
         value_trie_.MoveToNextSibling(&node)) {
      queue.push(node);
    }
  } while (!queue.empty());
}

void ValueDictionary::LookupPrefix(
    StringPiece key, bool use_kana_modifier_insensitive_lookup,
    Callback *callback) const {}

void ValueDictionary::LookupExact(StringPiece key, Callback *callback) const {
  if (key.empty()) {
    // For empty string, return nothing for compatibility reason; see the
    // comment above.
    return;
  }

  string lookup_key_str;
  codec_->EncodeValue(key, &lookup_key_str);
  if (value_trie_.ExactSearch(lookup_key_str) == -1) {
    return;
  }
  if (callback->OnKey(key) != Callback::TRAVERSE_CONTINUE) {
    return;
  }
  Token token;
  FillToken(suggestion_only_word_id_, key, &token);
  callback->OnToken(key, key, token);
}

void ValueDictionary::LookupReverse(StringPiece str,
                                    Callback *callback) const {}

}  // namespace dictionary
}  // namespace mozc