| // Copyright 2010-2015, Google Inc. |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: |
| // |
| // * Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above |
| // copyright notice, this list of conditions and the following disclaimer |
| // in the documentation and/or other materials provided with the |
| // distribution. |
| // * Neither the name of Google Inc. nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include <algorithm> |
| #include <cmath> |
| #include <iostream> // NOLINT |
| #include <iterator> |
| #include <vector> |
| #include <string> |
| |
| #include "base/file_stream.h" |
| #include "base/flags.h" |
| #include "base/logging.h" |
| #include "base/port.h" |
| #include "base/singleton.h" |
| #include "base/stopwatch.h" |
| #include "base/util.h" |
| #include "client/client.h" |
| #include "config/config_handler.h" |
| #include "session/commands.pb.h" |
| #include "session/random_keyevents_generator.h" |
| |
| DEFINE_string(server_path, "", "specify server path"); |
| DEFINE_string(log_path, "", "specify log output file path"); |
| |
| namespace mozc { |
| namespace { |
| |
| struct Result { |
| string test_name; |
| vector<uint32> operations_times; |
| }; |
| |
| class TestSentenceGenerator { |
| public: |
| const vector<vector<commands::KeyEvent> > &GetTestKeys() const { |
| return keys_; |
| } |
| |
| TestSentenceGenerator() { |
| size_t size = 0; |
| const char **sentences = |
| session::RandomKeyEventsGenerator::GetTestSentences(&size); |
| CHECK_GT(size, 0); |
| size = min(static_cast<size_t>(200), size); |
| |
| for (size_t i = 0; i < size; ++i) { |
| string output; |
| Util::HiraganaToRomanji(sentences[i], &output); |
| vector<commands::KeyEvent> tmp; |
| for (ConstChar32Iterator iter(output); !iter.Done(); iter.Next()) { |
| const char32 ucs4 = iter.Get(); |
| if (ucs4 >= static_cast<char32>('a') && |
| ucs4 <= static_cast<char32>('z')) { |
| commands::KeyEvent key; |
| key.set_key_code(static_cast<int>(ucs4)); |
| tmp.push_back(key); |
| } |
| } |
| if (!tmp.empty()) { |
| keys_.push_back(tmp); |
| } |
| } |
| } |
| |
| private: |
| vector<vector<commands::KeyEvent> > keys_; |
| }; |
| |
| class TestScenarioInterface { |
| public: |
| virtual void Run(Result *result) = 0; |
| |
| TestScenarioInterface() { |
| if (!FLAGS_server_path.empty()) { |
| client_.set_server_program(FLAGS_server_path); |
| } |
| CHECK(client_.IsValidRunLevel()) << "IsValidRunLevel failed"; |
| CHECK(client_.EnsureSession()) << "EnsureSession failed"; |
| CHECK(client_.NoOperation()) << "Server is not responding"; |
| } |
| |
| virtual ~TestScenarioInterface() {} |
| |
| protected: |
| virtual void IMEOn() { |
| commands::KeyEvent key; |
| key.set_special_key(commands::KeyEvent::ON); |
| client_.SendKey(key, &output_); |
| } |
| |
| virtual void IMEOff() { |
| commands::KeyEvent key; |
| key.set_special_key(commands::KeyEvent::OFF); |
| client_.SendKey(key, &output_); |
| } |
| |
| virtual void ResetConfig() { |
| config::Config config; |
| config::ConfigHandler::GetDefaultConfig(&config); |
| client_.SetConfig(config); |
| } |
| |
| virtual void EnableSuggestion() { |
| config::Config config; |
| config::ConfigHandler::GetDefaultConfig(&config); |
| config.set_use_history_suggest(true); |
| config.set_use_dictionary_suggest(true); |
| client_.SetConfig(config); |
| } |
| |
| virtual void DisableSuggestion() { |
| config::Config config; |
| config::ConfigHandler::GetDefaultConfig(&config); |
| config.set_use_history_suggest(false); |
| config.set_use_dictionary_suggest(false); |
| client_.SetConfig(config); |
| } |
| |
| client::Client client_; |
| commands::Output output_; |
| }; |
| |
| string GetBasicStats(const vector<uint32> times) { |
| uint32 total_time = 0; |
| uint32 avg_time = 0; |
| uint32 max_time = 0; |
| uint32 min_time = 0; |
| uint32 sd_time = 0; // Standard Deviation |
| uint32 med_time = 0; |
| |
| min_time = INT_MAX; |
| max_time = 0; |
| for (size_t i = 0; i < times.size(); ++i) { |
| total_time += times[i]; |
| min_time = min(times[i], min_time); |
| max_time = max(times[i], max_time); |
| } |
| |
| avg_time = static_cast<uint32>(1.0 * total_time / times.size()); |
| |
| if (times.size() >= 2) { |
| double dsd_time = 0; |
| for (size_t i = 0; i < times.size(); ++i) { |
| dsd_time += (avg_time - times[i]) * (avg_time - times[i]); |
| } |
| dsd_time = sqrt(dsd_time / (times.size() - 1)); |
| sd_time = static_cast<uint32>(dsd_time); |
| } |
| |
| if (!times.empty()) { |
| vector<uint32> tmp(times); |
| sort(tmp.begin(), tmp.end()); |
| med_time = tmp[tmp.size() / 2]; |
| } |
| |
| return Util::StringPrintf( |
| "size=%d total=%d avg=%d max=%d min=%d st=%d med=%d", |
| static_cast<int>(times.size()), |
| total_time, |
| avg_time, |
| max_time, |
| min_time, |
| sd_time, |
| med_time); |
| } |
| |
| class PreeditCommon : public TestScenarioInterface { |
| protected: |
| virtual void RunTest(Result *result) { |
| const vector<vector<commands::KeyEvent> > &keys = |
| Singleton<TestSentenceGenerator>::get()->GetTestKeys(); |
| for (size_t i = 0; i < keys.size(); ++i) { |
| for (int j = 0; j < keys[i].size(); ++j) { |
| Stopwatch stopwatch; |
| stopwatch.Start(); |
| client_.SendKey(keys[i][j], &output_); |
| stopwatch.Stop(); |
| result->operations_times.push_back(stopwatch.GetElapsedMicroseconds()); |
| } |
| commands::SessionCommand command; |
| command.set_type(commands::SessionCommand::REVERT); |
| client_.SendCommand(command, &output_); |
| } |
| } |
| }; |
| |
| class PreeditWithoutSuggestion : public PreeditCommon { |
| public: |
| virtual void Run(Result *result) { |
| result->test_name = "preedit_without_suggestion"; |
| ResetConfig(); |
| IMEOn(); |
| DisableSuggestion(); |
| RunTest(result); |
| IMEOff(); |
| ResetConfig(); |
| } |
| }; |
| |
| class PreeditWithSuggestion : public PreeditCommon { |
| public: |
| virtual void Run(Result *result) { |
| result->test_name = "preedit_with_suggestion"; |
| ResetConfig(); |
| IMEOn(); |
| EnableSuggestion(); |
| RunTest(result); |
| IMEOff(); |
| ResetConfig(); |
| } |
| }; |
| |
| enum PredictionRequestType { |
| ONE_CHAR, |
| TWO_CHARS |
| }; |
| |
| void CreatePredictionKeys(PredictionRequestType type, |
| vector <string> *request_keys) { |
| CHECK(request_keys); |
| request_keys->clear(); |
| |
| const char *kVoels[] = { "a", "i", "u", "e", "o" }; |
| const char *kConsonant[] = { "k", "s", "t", "n", "h", |
| "m", "y", "r", "w" }; |
| vector<string> one_chars; |
| for (size_t i = 0; i < arraysize(kVoels); ++i) { |
| one_chars.push_back(kVoels[i]); |
| } |
| |
| for (size_t i = 0; i < arraysize(kConsonant); ++i) { |
| for (size_t j = 0; j < arraysize(kVoels); ++j) { |
| one_chars.push_back(string(kConsonant[i]) + string(kVoels[j])); |
| } |
| } |
| |
| vector<string> two_chars; |
| for (size_t i = 0; i < one_chars.size(); ++i) { |
| for (size_t j = 0; j < one_chars.size(); ++j) { |
| two_chars.push_back(one_chars[i] + one_chars[j]); |
| } |
| } |
| switch (type) { |
| case ONE_CHAR: |
| copy(one_chars.begin(), one_chars.end(), |
| back_inserter(*request_keys)); |
| break; |
| case TWO_CHARS: |
| copy(two_chars.begin(), two_chars.end(), |
| back_inserter(*request_keys)); |
| break; |
| default: |
| break; |
| } |
| |
| CHECK(!request_keys->empty()); |
| } |
| |
| class PredictionCommon: public TestScenarioInterface { |
| protected: |
| void RunTest(PredictionRequestType type, Result *result) { |
| IMEOn(); |
| ResetConfig(); |
| DisableSuggestion(); |
| vector<string> request_keys; |
| CreatePredictionKeys(type, &request_keys); |
| for (size_t i = 0; i < request_keys.size(); ++i) { |
| const string &keys = request_keys[i]; |
| for (size_t j = 0; j < keys.size(); ++j) { |
| commands::KeyEvent key; |
| key.set_key_code(static_cast<int>(keys[j])); |
| client_.SendKey(key, &output_); |
| } |
| commands::KeyEvent key; |
| key.set_special_key(commands::KeyEvent::TAB); |
| Stopwatch stopwatch; |
| stopwatch.Start(); |
| client_.SendKey(key, &output_); |
| stopwatch.Stop(); |
| result->operations_times.push_back(stopwatch.GetElapsedMicroseconds()); |
| |
| commands::SessionCommand command; |
| command.set_type(commands::SessionCommand::REVERT); |
| client_.SendCommand(command, &output_); |
| } |
| IMEOff(); |
| } |
| }; |
| |
| class PredictionWithOneChar : public PredictionCommon { |
| public: |
| virtual void Run(Result *result) { |
| result->test_name = "prediction_one_char"; |
| RunTest(ONE_CHAR, result); |
| } |
| }; |
| |
| class PredictionWithTwoChars : public PredictionCommon { |
| public: |
| virtual void Run(Result *result) { |
| result->test_name = "prediction_two_chars"; |
| RunTest(TWO_CHARS, result); |
| } |
| }; |
| |
| class Conversion : public TestScenarioInterface { |
| public: |
| virtual void Run(Result *result) { |
| result->test_name = "conversion"; |
| ResetConfig(); |
| DisableSuggestion(); |
| IMEOn(); |
| |
| const vector<vector<commands::KeyEvent> > &keys = |
| Singleton<TestSentenceGenerator>::get()->GetTestKeys(); |
| for (size_t i = 0; i < keys.size(); ++i) { |
| for (int j = 0; j < keys[i].size(); ++j) { |
| client_.SendKey(keys[i][j], &output_); |
| } |
| commands::KeyEvent key; |
| key.set_special_key(commands::KeyEvent::SPACE); |
| Stopwatch stopwatch; |
| stopwatch.Start(); |
| client_.SendKey(key, &output_); |
| stopwatch.Stop(); |
| result->operations_times.push_back(stopwatch.GetElapsedMicroseconds()); |
| |
| commands::SessionCommand command; |
| command.set_type(commands::SessionCommand::REVERT); |
| client_.SendCommand(command, &output_); |
| } |
| |
| IMEOff(); |
| ResetConfig(); |
| } |
| }; |
| } // namespace |
| } // namespace mozc |
| |
| int main(int argc, char **argv) { |
| InitGoogle(argv[0], &argc, &argv, false); |
| |
| vector<mozc::TestScenarioInterface *> tests; |
| vector<mozc::Result *> results; |
| |
| tests.push_back(new mozc::PreeditWithoutSuggestion); |
| tests.push_back(new mozc::PreeditWithSuggestion); |
| tests.push_back(new mozc::Conversion); |
| tests.push_back(new mozc::PredictionWithOneChar); |
| tests.push_back(new mozc::PredictionWithTwoChars); |
| |
| for (size_t i = 0; i < tests.size(); ++i) { |
| mozc::Result *result = new mozc::Result; |
| tests[i]->Run(result); |
| results.push_back(result); |
| } |
| |
| CHECK_EQ(results.size(), tests.size()); |
| |
| ostream *ofs = &cout; |
| if (!FLAGS_log_path.empty()) { |
| ofs = new mozc::OutputFileStream(FLAGS_log_path.c_str()); |
| } |
| |
| // TODO(taku): generate histogram with ChartAPI |
| for (size_t i = 0; i < tests.size(); ++i) { |
| (*ofs) << results[i]->test_name << ": " << |
| mozc::GetBasicStats(results[i]->operations_times) << endl; |
| delete tests[i]; |
| delete results[i]; |
| } |
| if (ofs != &cout) { |
| delete ofs; |
| } |
| |
| return 0; |
| } |