| // Copyright 2011 Google Inc. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.enterprise.adaptor; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertTrue; |
| |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.rules.ExpectedException; |
| |
| import java.util.ArrayList; |
| import java.util.Collections; |
| |
| /** |
| * Tests for {@link Journal}. |
| */ |
| public class JournalTest { |
| @Rule |
| public ExpectedException thrown = ExpectedException.none(); |
| |
| @Test |
| public void testPushCounts() { |
| Journal journal = new Journal(new MockTimeProvider()); |
| DocId id = new DocId("id1"); |
| DocId id2 = new DocId("id2"); |
| DocId id3 = new DocId("id3"); |
| DocId id4 = new DocId("id4"); |
| ArrayList<DocIdPusher.Record> docs = new ArrayList<DocIdPusher.Record>(); |
| docs.add(new DocIdPusher.Record.Builder(id).build()); |
| docs.add(new DocIdPusher.Record.Builder(id2).build()); |
| docs.add(new DocIdPusher.Record.Builder(id3).build()); |
| journal.recordDocIdPush(docs); |
| assertEquals(3, journal.getSnapshot().numUniqueDocIdsPushed); |
| journal.recordDocIdPush(docs); |
| assertEquals(3, journal.getSnapshot().numUniqueDocIdsPushed); |
| docs.add(new DocIdPusher.Record.Builder(id4).build()); |
| journal.recordDocIdPush(docs); |
| assertEquals(4, journal.getSnapshot().numUniqueDocIdsPushed); |
| } |
| |
| @Test |
| public void testDisabledPushCounts() { |
| Journal journal = new Journal(true, new MockTimeProvider()); |
| journal.recordDocIdPush(Collections.singletonList( |
| new DocIdPusher.Record.Builder(new DocId("id")).build())); |
| assertEquals(-1, journal.getSnapshot().numUniqueDocIdsPushed); |
| } |
| |
| @Test |
| public void testUnsupportedDocIdPush() { |
| class UnsupportedItem implements DocIdSender.Item {}; |
| Journal journal = new Journal(true, new MockTimeProvider()); |
| thrown.expect(IllegalArgumentException.class); |
| journal.recordDocIdPush(Collections.singletonList(new UnsupportedItem())); |
| } |
| |
| @Test |
| public void testRequestCounts() { |
| Journal journal = new Journal(new MockTimeProvider()); |
| Journal.JournalSnapshot snapshot = journal.getSnapshot(); |
| assertEquals(0, snapshot.numUniqueGsaRequests); |
| assertEquals(0, snapshot.numTotalGsaRequests); |
| assertEquals(0, snapshot.numUniqueNonGsaRequests); |
| assertEquals(0, snapshot.numTotalNonGsaRequests); |
| |
| DocId doc1 = new DocId("1"); |
| DocId doc2 = new DocId("2"); |
| |
| journal.recordGsaContentRequest(doc1); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.numUniqueGsaRequests); |
| assertEquals(1, snapshot.numTotalGsaRequests); |
| |
| journal.recordGsaContentRequest(doc1); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.numUniqueGsaRequests); |
| assertEquals(2, snapshot.numTotalGsaRequests); |
| |
| journal.recordGsaContentRequest(doc2); |
| snapshot = journal.getSnapshot(); |
| assertEquals(2, snapshot.numUniqueGsaRequests); |
| assertEquals(3, snapshot.numTotalGsaRequests); |
| |
| assertEquals(0, snapshot.numUniqueNonGsaRequests); |
| assertEquals(0, snapshot.numTotalNonGsaRequests); |
| |
| journal.recordNonGsaContentRequest(doc1); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.numUniqueNonGsaRequests); |
| assertEquals(1, snapshot.numTotalNonGsaRequests); |
| |
| journal.recordNonGsaContentRequest(doc1); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.numUniqueNonGsaRequests); |
| assertEquals(2, snapshot.numTotalNonGsaRequests); |
| |
| journal.recordNonGsaContentRequest(doc2); |
| snapshot = journal.getSnapshot(); |
| assertEquals(2, snapshot.numUniqueNonGsaRequests); |
| assertEquals(3, snapshot.numTotalNonGsaRequests); |
| |
| assertEquals(2, snapshot.numUniqueGsaRequests); |
| assertEquals(3, snapshot.numTotalGsaRequests); |
| } |
| |
| @Test |
| public void testStats() throws InterruptedException { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| timeProvider.autoIncrement = false; |
| timeProvider.time |
| = journal.getSnapshot().timeStats[0].pendingStatPeriodEnd; |
| journal.recordRequestProcessingStart(); |
| Thread thread = new Thread() { |
| public void run() { |
| timeProvider.time += 2; |
| journal.recordRequestProcessingStart(); |
| timeProvider.time += 2; |
| journal.recordRequestProcessingEnd(10); |
| } |
| }; |
| thread.start(); |
| thread.join(); |
| journal.recordRequestProcessingEnd(8); |
| Journal.JournalSnapshot snapshot = journal.getSnapshot(); |
| Journal.Stat stat |
| = snapshot.timeStats[0].stats[snapshot.timeStats[0].currentStat]; |
| assertEquals(2, stat.requestProcessingsCount); |
| assertEquals(6, stat.requestProcessingsDurationSum); |
| assertEquals(4, stat.requestProcessingsMaxDuration); |
| assertEquals(18, stat.requestProcessingsThroughput); |
| |
| // Did it swap out to a new stat correctly? |
| journal.recordRequestProcessingStart(); |
| long previousTime = timeProvider.time; |
| timeProvider.time |
| = journal.getSnapshot().timeStats[0].pendingStatPeriodEnd; |
| journal.recordRequestProcessingEnd(101); |
| snapshot = journal.getSnapshot(); |
| stat = snapshot.timeStats[0].stats[snapshot.timeStats[0].currentStat]; |
| assertEquals(1, stat.requestProcessingsCount); |
| assertEquals(timeProvider.time - previousTime, |
| stat.requestProcessingsDurationSum); |
| assertEquals(timeProvider.time - previousTime, |
| stat.requestProcessingsMaxDuration); |
| assertEquals(101, stat.requestProcessingsThroughput); |
| |
| // Does it still have correct values for previous stat? |
| stat = snapshot.timeStats[0].stats[(snapshot.timeStats[0].currentStat - 1) |
| % snapshot.timeStats[0].stats.length]; |
| assertEquals(2, stat.requestProcessingsCount); |
| assertEquals(6, stat.requestProcessingsDurationSum); |
| assertEquals(4, stat.requestProcessingsMaxDuration); |
| assertEquals(18, stat.requestProcessingsThroughput); |
| |
| // Did it clear out everything after a long time? |
| // Subtract off a reasonable amount of time to prevevent overflow. |
| timeProvider.time = Long.MAX_VALUE - 1000 * 60 * 60 * 24 * 7; |
| journal.recordRequestProcessingStart(); |
| journal.recordRequestProcessingEnd(1); |
| snapshot = journal.getSnapshot(); |
| for (int i = 0; i < snapshot.timeStats[0].stats.length; i++) { |
| if (i == snapshot.timeStats[0].currentStat) { |
| continue; |
| } |
| stat = snapshot.timeStats[0].stats[i]; |
| assertEquals(0, stat.requestProcessingsCount); |
| assertEquals(0, stat.requestProcessingsDurationSum); |
| assertEquals(0, stat.requestProcessingsMaxDuration); |
| assertEquals(0, stat.requestProcessingsThroughput); |
| } |
| } |
| |
| @Test |
| public void testFullPushStats() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| timeProvider.autoIncrement = false; |
| |
| timeProvider.time = 1; |
| journal.recordFullPushStarted(); |
| timeProvider.time = 2; |
| journal.recordFullPushSuccessful(); |
| Journal.JournalSnapshot snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulFullPushStart); |
| assertEquals(2, snapshot.lastSuccessfulFullPushEnd); |
| |
| timeProvider.time = 5; |
| journal.recordFullPushStarted(); |
| timeProvider.time = 7; |
| journal.recordFullPushInterrupted(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulFullPushStart); |
| assertEquals(2, snapshot.lastSuccessfulFullPushEnd); |
| |
| timeProvider.time = 8; |
| journal.recordFullPushStarted(); |
| timeProvider.time = 9; |
| journal.recordFullPushFailed(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulFullPushStart); |
| assertEquals(2, snapshot.lastSuccessfulFullPushEnd); |
| |
| timeProvider.time = 11; |
| journal.recordFullPushStarted(); |
| timeProvider.time = 15; |
| journal.recordFullPushSuccessful(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(11, snapshot.lastSuccessfulFullPushStart); |
| assertEquals(15, snapshot.lastSuccessfulFullPushEnd); |
| } |
| |
| @Test |
| public void testFullPushStartDouble() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| journal.recordFullPushStarted(); |
| thrown.expect(IllegalStateException.class); |
| journal.recordFullPushStarted(); |
| } |
| |
| @Test |
| public void testFullPushInterruptedPremature() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| thrown.expect(IllegalStateException.class); |
| journal.recordFullPushInterrupted(); |
| } |
| |
| @Test |
| public void testFullPushFailedPremature() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| thrown.expect(IllegalStateException.class); |
| journal.recordFullPushFailed(); |
| } |
| |
| @Test |
| public void testIncrementalPushStats() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| timeProvider.autoIncrement = false; |
| |
| timeProvider.time = 1; |
| journal.recordIncrementalPushStarted(); |
| timeProvider.time = 2; |
| journal.recordIncrementalPushSuccessful(); |
| Journal.JournalSnapshot snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulIncrementalPushStart); |
| assertEquals(2, snapshot.lastSuccessfulIncrementalPushEnd); |
| |
| timeProvider.time = 5; |
| journal.recordIncrementalPushStarted(); |
| timeProvider.time = 7; |
| journal.recordIncrementalPushInterrupted(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulIncrementalPushStart); |
| assertEquals(2, snapshot.lastSuccessfulIncrementalPushEnd); |
| |
| timeProvider.time = 8; |
| journal.recordIncrementalPushStarted(); |
| timeProvider.time = 9; |
| journal.recordIncrementalPushFailed(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(1, snapshot.lastSuccessfulIncrementalPushStart); |
| assertEquals(2, snapshot.lastSuccessfulIncrementalPushEnd); |
| |
| timeProvider.time = 11; |
| journal.recordIncrementalPushStarted(); |
| timeProvider.time = 15; |
| journal.recordIncrementalPushSuccessful(); |
| snapshot = journal.getSnapshot(); |
| assertEquals(11, snapshot.lastSuccessfulIncrementalPushStart); |
| assertEquals(15, snapshot.lastSuccessfulIncrementalPushEnd); |
| } |
| |
| @Test |
| public void testIncrementalPushStartDouble() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| journal.recordIncrementalPushStarted(); |
| thrown.expect(IllegalStateException.class); |
| journal.recordIncrementalPushStarted(); |
| } |
| |
| @Test |
| public void testIncrementalPushInterruptedPremature() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| thrown.expect(IllegalStateException.class); |
| journal.recordIncrementalPushInterrupted(); |
| } |
| |
| @Test |
| public void testIncrementalPushFailedPremature() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| // Make sure we are past epoch, because that is a special value in the |
| // journaling code. |
| timeProvider.time = 1; |
| thrown.expect(IllegalStateException.class); |
| journal.recordIncrementalPushFailed(); |
| } |
| @Test |
| public void testStatsNoStart() { |
| Journal journal = new Journal(new MockTimeProvider()); |
| thrown.expect(IllegalStateException.class); |
| journal.recordRequestProcessingEnd(1); |
| } |
| |
| @Test |
| public void testDefaultTimeProvider() { |
| new Journal(false); |
| } |
| |
| @Test |
| public void testLastFullPushStatus() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| |
| assertEquals(Journal.CompletionStatus.SUCCESS, |
| journal.getLastFullPushStatus()); |
| |
| journal.recordFullPushStarted(); |
| journal.recordFullPushInterrupted(); |
| assertEquals(Journal.CompletionStatus.INTERRUPTION, |
| journal.getLastFullPushStatus()); |
| |
| journal.recordFullPushStarted(); |
| journal.recordFullPushFailed(); |
| assertEquals(Journal.CompletionStatus.FAILURE, |
| journal.getLastFullPushStatus()); |
| |
| journal.recordFullPushStarted(); |
| journal.recordFullPushSuccessful(); |
| assertEquals(Journal.CompletionStatus.SUCCESS, |
| journal.getLastFullPushStatus()); |
| } |
| |
| @Test |
| public void testLastIncrementalPushStatus() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| |
| assertEquals(Journal.CompletionStatus.SUCCESS, |
| journal.getLastIncrementalPushStatus()); |
| |
| journal.recordIncrementalPushStarted(); |
| journal.recordIncrementalPushInterrupted(); |
| assertEquals(Journal.CompletionStatus.INTERRUPTION, |
| journal.getLastIncrementalPushStatus()); |
| |
| journal.recordIncrementalPushStarted(); |
| journal.recordIncrementalPushFailed(); |
| assertEquals(Journal.CompletionStatus.FAILURE, |
| journal.getLastIncrementalPushStatus()); |
| |
| journal.recordIncrementalPushStarted(); |
| journal.recordIncrementalPushSuccessful(); |
| assertEquals(Journal.CompletionStatus.SUCCESS, |
| journal.getLastIncrementalPushStatus()); |
| } |
| |
| @Test |
| public void testRetrieverStatusSource() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| timeProvider.autoIncrement = false; |
| DocId docId = new DocId(""); |
| final long hourInMillis = 1000 * 60 * 60; |
| |
| double rate = journal.getRetrieverErrorRate(10); |
| assertEquals(0., rate, 0.); |
| |
| journal.recordRequestProcessingStart(); |
| journal.recordRequestProcessingFailure(); |
| rate = journal.getRetrieverErrorRate(10); |
| assertEquals(1., rate, 0.); |
| |
| timeProvider.time += hourInMillis; |
| for (int i = 0; i < 7; i++) { |
| journal.recordRequestProcessingStart(); |
| journal.recordRequestProcessingEnd(0); |
| } |
| rate = journal.getRetrieverErrorRate(10); |
| assertEquals(1. / 8., rate, 0.); |
| |
| timeProvider.time += hourInMillis; |
| for (int i = 0; i < 7; i++) { |
| journal.recordRequestProcessingStart(); |
| journal.recordRequestProcessingFailure(); |
| } |
| rate = journal.getRetrieverErrorRate(10); |
| assertEquals(1. / 2., rate, 0.); |
| rate = journal.getRetrieverErrorRate(7); |
| assertEquals(1., rate, 0.); |
| |
| timeProvider.time += hourInMillis; |
| for (int i = 0; i < 10; i++) { |
| journal.recordRequestProcessingStart(); |
| journal.recordRequestProcessingEnd(0); |
| } |
| rate = journal.getRetrieverErrorRate(10); |
| assertEquals(0., rate, 0.); |
| } |
| |
| @Test |
| public void testGsaCrawlingStatusSource() { |
| final MockTimeProvider timeProvider = new MockTimeProvider(); |
| final Journal journal = new Journal(timeProvider); |
| timeProvider.autoIncrement = false; |
| DocId docId = new DocId(""); |
| |
| assertFalse(journal.hasGsaCrawledWithinLastDay()); |
| |
| journal.recordGsaContentRequest(docId); |
| assertTrue(journal.hasGsaCrawledWithinLastDay()); |
| |
| final long dayInMillis = 1000 * 60 * 60 * 24; |
| timeProvider.time += 2 * dayInMillis; |
| assertFalse(journal.hasGsaCrawledWithinLastDay()); |
| } |
| } |