Incremental Crawl Phase 2 This makes incremental crawl more efficient.
diff --git a/src/com/google/enterprise/adaptor/ad/AdAdaptor.java b/src/com/google/enterprise/adaptor/ad/AdAdaptor.java index b18118f..0a0c899 100644 --- a/src/com/google/enterprise/adaptor/ad/AdAdaptor.java +++ b/src/com/google/enterprise/adaptor/ad/AdAdaptor.java
@@ -30,6 +30,7 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -187,12 +188,14 @@ clearLastCompleteGroupCatalog(); GroupCatalog cumulativeCatalog = makeFullCatalog(); // all servers were able to successfully populate the catalog: do a push - cumulativeCatalog.resolveForeignSecurityPrincipals(); + // TODO(myk): Rework the structure so that a member variable of + // cumulativeCatalog isn't passed in as a parameter to its own method. + cumulativeCatalog.resolveForeignSecurityPrincipals( + cumulativeCatalog.entities); Map<GroupPrincipal, List<Principal>> groups = - cumulativeCatalog.makeDefs(); + cumulativeCatalog.makeDefs(cumulativeCatalog.entities); pusher.pushGroupDefinitions(groups, CASE_SENSITIVITY); - // TODO(myk): clear membership information from cache - retain only the - // entities in bySid, byDn, and wellKnownMembership. + cumulativeCatalog.members.clear(); lastCompleteGroupCatalog = cumulativeCatalog; } finally { mutex.unlock(); @@ -200,8 +203,8 @@ } } - private GroupCatalog makeFullCatalog() throws InterruptedException, - IOException { + @VisibleForTesting + GroupCatalog makeFullCatalog() throws InterruptedException, IOException { GroupCatalog cumulativeCatalog = new GroupCatalog(localizedStrings, namespace, feedBuiltinGroups); for (AdServer server : servers) { @@ -209,7 +212,7 @@ server.ensureConnectionIsCurrent(); GroupCatalog catalog = new GroupCatalog(localizedStrings, namespace, feedBuiltinGroups); - catalog.readEverythingFrom(server); + catalog.readEverythingFrom(server, /*includeMembers=*/ true); cumulativeCatalog.add(catalog); } catch (NamingException ne) { String host = server.getHostName(); @@ -224,8 +227,7 @@ * <p> * When a server cannot do an incremental push, it does a full crawl without * doing a push afterwards -- this sets up its state so that subsequent - * incremental pushes can work. A future version of this method will do the - * full crawl ignoring the "member" attribute. + * incremental pushes can work. */ @Override public void getModifiedDocIds(DocIdPusher pusher) throws InterruptedException, @@ -237,39 +239,50 @@ } try { log.log(Level.FINE, "getModifiedDocIds starting - acquired lock."); - - for (AdServer server : servers) { - String previousServiceName = server.getDsServiceName(); - String previousInvocationId = server.getInvocationID(); - long previousHighestUSN = server.getHighestCommittedUSN(); - try { - server.ensureConnectionIsCurrent(); - // TODO(myk): combine each server's resulting Entities into one Set - lastCompleteGroupCatalog.readUpdatesFrom(server, previousServiceName, - previousInvocationId, previousHighestUSN); - } catch (NamingException ne) { - // invalidate the saved group catalog - clearLastCompleteGroupCatalog(); - String host = server.getHostName(); - throw new IOException("could not get entities from " + host, ne); - } - } - - // all servers were able to successfully update the catalog: do a push - lastCompleteGroupCatalog.resolveForeignSecurityPrincipals(); - Map<GroupPrincipal, List<Principal>> groups = - lastCompleteGroupCatalog.makeDefs(); - // TODO(myk): pass on (only) new/modified entities to resolve/makeDefs, - // so that we're only pushing the new/modified groups below. - pusher.pushGroupDefinitions(groups, CASE_SENSITIVITY); - // TODO(myk): clear membership information from cache - retain only the - // entities in bySid, byDn, and wellKnownMembership. + getModifiedDocIdsHelper(pusher); } finally { mutex.unlock(); log.log(Level.FINE, "getModifiedDocIds ending - lock released."); } } + @VisibleForTesting + void getModifiedDocIdsHelper(DocIdPusher pusher) throws InterruptedException, + IOException { + if (lastCompleteGroupCatalog == null) { + log.log(Level.FINE, "getModifiedDocIds doing a fetch with no push."); + lastCompleteGroupCatalog = makeFullCatalog(); + return; + } + + Set<AdEntity> allNewOrUpdatedEntities = new HashSet<AdEntity>(); + for (AdServer server : servers) { + String previousServiceName = server.getDsServiceName(); + String previousInvocationId = server.getInvocationID(); + long previousHighestUSN = server.getHighestCommittedUSN(); + try { + server.ensureConnectionIsCurrent(); + allNewOrUpdatedEntities.addAll( + lastCompleteGroupCatalog.readUpdatesFrom(server, + previousServiceName, previousInvocationId, + previousHighestUSN)); + } catch (NamingException ne) { + // invalidate the saved group catalog + clearLastCompleteGroupCatalog(); + String host = server.getHostName(); + throw new IOException("could not get entities from " + host, ne); + } + } + + // all servers were able to successfully update the catalog: do a push + lastCompleteGroupCatalog.resolveForeignSecurityPrincipals( + allNewOrUpdatedEntities); + Map<GroupPrincipal, List<Principal>> groups = + lastCompleteGroupCatalog.makeDefs(allNewOrUpdatedEntities); + pusher.pushGroupDefinitions(groups, CASE_SENSITIVITY); + lastCompleteGroupCatalog.members.clear(); + } + // don't expose the <code>lastCompleteGroupCatalog</code> field, but do allow // tests to clear it @VisibleForTesting @@ -357,9 +370,14 @@ } @VisibleForTesting - void readEverythingFrom(AdServer server) throws InterruptedNamingException { - // TODO(myk): Phase II: indicate whether or not this search should - // include members + void readEverythingFrom(AdServer server, boolean includeMembers) + throws InterruptedNamingException { + final String[] nonMemberAttributes = new String[] { "uSNChanged", + "sAMAccountName", "objectGUID;binary", "objectSid;binary", + "userPrincipalName", "primaryGroupId", "userAccountControl" }; + final String[] allAttributes = Arrays.copyOf(nonMemberAttributes, + nonMemberAttributes.length + 1); + allAttributes[nonMemberAttributes.length] = "member"; log.log(Level.FINE, "Starting full crawl."); // LDAP_MATCHING_RULE_BIT_AND = 1.2.840.113556.1.4.803 // and ADS_GROUP_TYPE_SECURITY_ENABLED = 2147483648. @@ -367,9 +385,7 @@ + "(groupType:1.2.840.113556.1.4.803:=2147483648))" + "(&(objectClass=user)(objectCategory=person)))", /*deleted=*/ false, - new String[] { "uSNChanged", "sAMAccountName", "objectGUID;binary", - "objectSid;binary", "userPrincipalName", "primaryGroupId", - "member", "userAccountControl" }); + includeMembers ? allAttributes : nonMemberAttributes); // disabled groups handled later, in makeDefs() log.log(Level.FINE, "Ending full crawl - now starting processing."); processEntities(entities, server.getnETBIOSName()); @@ -392,14 +408,13 @@ * @param previousHighestUSN last-crawled value of * <code>server.getHighestCommittedUSN()</code> * <code>previousHighestUSN</code>. - */ - /* TODO(myk): add + * * @return all instances of <code>AdEntity</code> that are users/groups that * have a <code>uSNChanged</code> attribute newer than, or * <code>Collections.emptySet()</code> when the cache had been stale. */ @VisibleForTesting - void readUpdatesFrom(AdServer server, String previousServiceName, + Set<AdEntity> readUpdatesFrom(AdServer server, String previousServiceName, String previousInvocationId, long previousHighestUSN) throws InterruptedNamingException { // TODO(myk): Determine whether adaptors should include code to get/set @@ -417,26 +432,23 @@ + "partial updates support.", new Object[]{previousServiceName, currentServiceName}); } - readEverythingFrom(server); - return; - //TODO(myk): return Collections.emptySet(); + readEverythingFrom(server, /*includeMembers=*/ false); + return Collections.emptySet(); } if (!currentInvocationId.equals(previousInvocationId)) { log.log(Level.WARNING, "Directory Controller {0} has been restored from backup. " + "Performing full recrawl.", currentServiceName); - readEverythingFrom(server); - return; - //TODO(myk): return Collections.emptySet(); + readEverythingFrom(server, /*includeMembers=*/ false); + return Collections.emptySet(); } if (currentHighestUSN == previousHighestUSN) { log.log(Level.INFO, "No updates on server {0} -- no crawl invoked.", server); - return; - //TODO(myk): return Collections.emptySet(); + return Collections.emptySet(); } log.log(Level.INFO, "Attempting incremental crawl."); - incrementalCrawl(server, previousHighestUSN, currentHighestUSN); + return incrementalCrawl(server, previousHighestUSN, currentHighestUSN); } private void processEntities(Set<AdEntity> entities, String nETBIOSName) { @@ -459,7 +471,6 @@ log.log(Level.FINE, "Starting incremental crawl."); // LDAP_MATCHING_RULE_BIT_AND = 1.2.840.113556.1.4.803 // and ADS_GROUP_TYPE_SECURITY_ENABLED = 2147483648. - // TODO(myk): Phase II: indicate that this search should exclude members Set<AdEntity> newOrModifiedEntities = server.search("(&(uSNChanged>=" + (previousHighestUSN + 1) + ")(|(&(objectClass=group)" + "(groupType:1.2.840.113556.1.4.803:=2147483648))" @@ -534,9 +545,7 @@ log.log(Level.FINE, "# users added to all primary groups: {0}", nadds); } - void resolveForeignSecurityPrincipals() { - //TODO(myk) Phase II: pass in only the entities just read in (for an - // incremental search to only consider those entities) + void resolveForeignSecurityPrincipals(Set<AdEntity> entities) { int nGroups = 0; int nNullSid = 0; int nNullResolution = 0; @@ -572,10 +581,8 @@ log.log(Level.FINE, "#resolved: {0}", nResolved); } - Map<GroupPrincipal, List<Principal>> makeDefs() { + Map<GroupPrincipal, List<Principal>> makeDefs(Set<AdEntity> entities) { // Merge members with well known group members - //TODO(myk) Phase II: pass in only the entities just read in (for an - // incremental search to only consider those entities) Map<AdEntity, Set<String>> allMembers = new HashMap<AdEntity, Set<String>>(members); allMembers.putAll(wellKnownMembership);
diff --git a/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java b/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java index 14b4228..8a3c507 100644 --- a/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java +++ b/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java
@@ -50,10 +50,15 @@ import java.util.logging.Level; import java.util.logging.Logger; +import javax.naming.CommunicationException; import javax.naming.InterruptedNamingException; +import javax.naming.NamingException; + /** Test cases for {@link AdAdaptor}. */ public class AdAdaptorTest { + public static final int BRIEF_DELAY_IN_MILLISECONDS = 25; + @Test public void testNoop() { } @@ -116,8 +121,13 @@ groupCatalog.add(golden); groupCatalog.domain.clear(); assertFalse(golden.equals(groupCatalog)); - assertFalse(golden.hashCode() == groupCatalog.hashCode()); + + almostClear(groupCatalog); + groupCatalog.add(golden); + assertEquals(golden, groupCatalog); + groupCatalog.wellKnownMembership.get(groupCatalog.everyone).add("fakeDN"); + assertFalse(golden.equals(groupCatalog)); } @Test @@ -145,7 +155,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); final AdEntity goldenEntity = new AdEntity("S-1-0-0", "cn=name\\ under,DN_for_default_naming_context"); @@ -199,7 +209,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); AdEntity[] groupEntity = groupCatalog.entities.toArray(new AdEntity[0]); final AdEntity goldenEntity = groupEntity[0]; @@ -261,7 +271,7 @@ adServer.initialize(); groupCatalog.bySid.put("S-1-5-32-users", userGroup); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); final AdEntity goldenEntity = new AdEntity("S-1-5-32-544", "cn=name\\ under,DN_for_default_naming_context", "users", "sam"); @@ -288,7 +298,7 @@ assertTrue(golden.equals(groupCatalog)); // make sure readEverythingFrom call is idempotent - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); assertTrue(golden.equals(groupCatalog)); } @@ -315,7 +325,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); final AdEntity goldenEntity = new AdEntity("S-1-5-32-544", "cn=name\\ under,DN_for_default_naming_context", "users", "sam"); @@ -339,27 +349,24 @@ assertTrue(golden.equals(groupCatalog)); // make sure readEverythingFrom call is idempotent - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); assertTrue(golden.equals(groupCatalog)); } @Test public void testFullCrawlVersusIncrementalCrawlFlow() throws Exception { - FakeAdaptor adAdaptor = new FakeAdaptor(); - FakeCatalog groupCatalog = new FakeCatalog(defaultLocalizedStringMap(), - "example.com", false); + final FakeAdaptor adAdaptor = new FakeAdaptor(); + final FakeCatalog groupCatalog = new FakeCatalog( + defaultLocalizedStringMap(), "example.com", false); MockLdapContext ldapContext = defaultMockLdapContext(); - AdServer adServer = new AdServer("localhost", ldapContext); + final AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - // the following 3 lines initialize AdAdAptor. - AccumulatingDocIdPusher pusher = new AccumulatingDocIdPusher(); - Map<String, String> configEntries = defaultConfig(); groupCatalog.resetCrawlFlags(); assertFalse(groupCatalog.ranFullCrawl()); assertFalse(groupCatalog.ranIncrementalCrawl()); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); assertTrue(groupCatalog.ranFullCrawl()); assertFalse(groupCatalog.ranIncrementalCrawl()); @@ -381,13 +388,44 @@ assertFalse(groupCatalog.ranFullCrawl()); assertFalse(groupCatalog.ranIncrementalCrawl()); - // call Incremental call (only) when it's desired and the service name and - // invocation ID both match and the HighestUSN does not match. groupCatalog.resetCrawlFlags(); groupCatalog.readUpdatesFrom(adServer, "ds_service_name", "0x0123456789abc", - 12345677L); + 12345677L); // earlier USN than previous run: does an incremental run assertFalse(groupCatalog.ranFullCrawl()); assertTrue(groupCatalog.ranIncrementalCrawl()); + + // attempt to invoke incremental crawl during a full crawl + MoreFakeAdaptor adaptor = new MoreFakeAdaptor(); + adaptor.resetCrawlFlags(); + Thread fullThread = new FullCrawlThread(adaptor); + Thread incrementalThread = new IncrementalCrawlThread(adaptor); + fullThread.start(); + try { + Thread.sleep(BRIEF_DELAY_IN_MILLISECONDS); + } catch (InterruptedException ex) { + throw new InterruptedNamingException(ex.getMessage()); + } + incrementalThread.start(); + fullThread.join(); + incrementalThread.join(); + assertTrue(adaptor.ranFullCrawl()); + assertFalse(adaptor.ranIncrementalCrawl()); + + // invoke full crawl during an incremental crawl + incrementalThread = new IncrementalCrawlThread(adaptor); + fullThread = new FullCrawlThread(adaptor); + adaptor.resetCrawlFlags(); + incrementalThread.start(); + try { + Thread.sleep(BRIEF_DELAY_IN_MILLISECONDS); + } catch (InterruptedException ex) { + throw new InterruptedNamingException(ex.getMessage()); + } + fullThread.start(); + incrementalThread.join(); + fullThread.join(); + assertTrue(adaptor.ranFullCrawl()); + assertTrue(adaptor.ranIncrementalCrawl()); } @Test @@ -436,16 +474,22 @@ adServer.initialize(); // first, do a full crawl - groupCatalog.readEverythingFrom(adServer); + Set<AdEntity> updateResults = groupCatalog.readUpdatesFrom(adServer, null, + "0x0123456789abc", 12345677L); + Set<AdEntity> goldenResults = Collections.emptySet(); + assertEquals(goldenResults, updateResults); // now do an incremental crawl - groupCatalog.readUpdatesFrom(adServer, "ds_service_name", "0x0123456789abc", - 12345677L); + updateResults = groupCatalog.readUpdatesFrom(adServer, "ds_service_name", + "0x0123456789abc", 12345677L); // extract incrementally-added user as one golden entity Set<AdEntity> incrementalUserSet = adServer.search(incrementalFilter, false, new String[] { "member", "objectSid;binary", "objectGUID;binary", "primaryGroupId", "sAMAccountName" }); + goldenResults = incrementalUserSet; + assertEquals(goldenResults, updateResults); + assertEquals(1, incrementalUserSet.size()); for (AdEntity ae : incrementalUserSet) { assertFalse(ae.isGroup()); @@ -464,7 +508,6 @@ AdEntity[] groupEntity = fullyCrawledGroupSet.toArray(new AdEntity[0]); final AdEntity goldenGroupEntity = new AdEntity("S-1-0-0", "cn=name\\ under,DN_for_default_naming_context"); - goldenGroupEntity.getMembers().addAll(members); final Map<AdEntity, Set<String>> goldenMembers = new HashMap<AdEntity, Set<String>>(); @@ -487,6 +530,12 @@ /*bySid*/ goldenSid, /*byDn*/ goldenDn, /*domain*/ goldenDomain); + assertEquals(golden, groupCatalog); + + // do another incremental crawl with same results + updateResults = groupCatalog.readUpdatesFrom(adServer, "ds_service_name", + "0x0123456789abc", 12345677L); + assertEquals(goldenResults, updateResults); } @Test @@ -526,7 +575,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); // add two additional entities to test all branches of our method. // first -- a user @@ -545,7 +594,7 @@ assertTrue(wellKnownEntity.isWellKnown()); groupCatalog.entities.add(wellKnownEntity); - groupCatalog.resolveForeignSecurityPrincipals(); + groupCatalog.resolveForeignSecurityPrincipals(groupCatalog.entities); // extract original group entity Set<AdEntity> groupEntitySet = adServer.search(filter, false, @@ -592,7 +641,7 @@ assertTrue(golden.equals(groupCatalog)); // make sure resolveForeignSecurityPrincipals call is idempotent - groupCatalog.resolveForeignSecurityPrincipals(); + groupCatalog.resolveForeignSecurityPrincipals(groupCatalog.entities); assertTrue(golden.equals(groupCatalog)); } @@ -608,7 +657,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); tweakGroupCatalogForMakeDefs(groupCatalog, adServer, false); @@ -622,7 +671,7 @@ golden.put(new GroupPrincipal("known_group", "example.com"), Collections.<Principal>emptyList()); } - assertEquals(golden, groupCatalog.makeDefs()); + assertEquals(golden, groupCatalog.makeDefs(groupCatalog.entities)); } @Test @@ -637,7 +686,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); tweakGroupCatalogForMakeDefs(groupCatalog, adServer, true); @@ -649,7 +698,7 @@ golden.put(new GroupPrincipal("known_group", "example.com"), Collections.<Principal>emptyList()); } - assertEquals(golden, groupCatalog.makeDefs()); + assertEquals(golden, groupCatalog.makeDefs(groupCatalog.entities)); } @Test @@ -676,7 +725,7 @@ AdServer adServer = new AdServer("localhost", ldapContext); adServer.initialize(); - groupCatalog.readEverythingFrom(adServer); + groupCatalog.readEverythingFrom(adServer, /*includeMembers=*/ true); tweakGroupCatalogForMakeDefs(groupCatalog, adServer, false); // now replace the parent group with a well-known one @@ -687,7 +736,7 @@ if ("cn=name\\ under,DN_for_default_naming_context".equals(entity.getDn()) && (entity.getMembers().size() == 4)) { formerGroup = entity; - for (String member: entity.getMembers()) { + for (String member : entity.getMembers()) { replacementGroup.getMembers().add(member); } // trigger the IllegalArgumentException paths by adding empty-named @@ -726,7 +775,7 @@ golden.put(new GroupPrincipal("known_group", "example.com"), Collections.<Principal>emptyList()); } - assertEquals(golden, groupCatalog.makeDefs()); + assertEquals(golden, groupCatalog.makeDefs(groupCatalog.entities)); } // Tests for the methods of the outer class @@ -759,7 +808,8 @@ configEntries.put("ad.ldapReadTimeoutSecs", ""); configEntries.put("server.port", "5680"); configEntries.put("server.dashboardPort", "5681"); - pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true); + pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true, + /*init=*/ true); Map<GroupPrincipal, Collection<Principal>> results = pusher.getGroups(); // the above (eventually) calls AdAdaptor.init() with the specified config. } @@ -784,7 +834,8 @@ configEntries.put("ad.ldapReadTimeoutSecs", "0"); configEntries.put("server.port", "5680"); configEntries.put("server.dashboardPort", "5681"); - pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true); + pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true, + /*init=*/ true); Map<GroupPrincipal, Collection<Principal>> results = pusher.getGroups(); // the above (eventually) calls AdAdaptor.init() with the specified config. } @@ -875,7 +926,8 @@ AdAdaptor adAdaptor = new FakeAdaptor(); AccumulatingDocIdPusher pusher = new AccumulatingDocIdPusher(); Map<String, String> configEntries = defaultConfig(); - pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true); + pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true, + /*init=*/ true); Map<GroupPrincipal, Collection<Principal>> results = pusher.getGroups(); final Map<GroupPrincipal, Collection<Principal>> goldenGroups = @@ -893,12 +945,14 @@ assertEquals(goldenGroups, results); // make sure pushGroupDefinitions call is idempotent - pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true); + pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ true, + /*init=*/ false); results = pusher.getGroups(); assertEquals(goldenGroups, results); // even when doing an incremental push - pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ false); + pushGroupDefinitions(adAdaptor, configEntries, pusher, /*fullPush=*/ false, + /*init=*/ false); results = pusher.getGroups(); assertEquals(goldenGroups, results); } @@ -919,18 +973,25 @@ fail("Could not create LdapContext:" + e); } return new AdServer(host, ldapContext) { + int timesSearchCalled = 0; + int timesEnsureConnectionCalled = 0; @Override public Set<AdEntity> search(String filter, boolean deleted, String[] attributes) throws InterruptedNamingException { - if (errorFilter.equals(filter)) { - throw new InterruptedNamingException("Catch me if you can!"); + if (errorFilter.equals(filter) && timesSearchCalled++ == 0) { + throw new InterruptedNamingException("First exception"); } else { return super.search(filter, deleted, attributes); } } @Override - void recreateLdapContext() { - // leave ldapContext unchanged + public void ensureConnectionIsCurrent() + throws CommunicationException, NamingException { + if (timesEnsureConnectionCalled++ < 9) { + super.ensureConnectionIsCurrent(); + } else { + throw new InterruptedNamingException("Second exception"); + } } }; } @@ -939,14 +1000,39 @@ Map<String, String> configEntries = defaultConfig(); try { pushGroupDefinitions(adAdaptor, configEntries, pusher, - /*fullPush=*/ true); + /*fullPush=*/ true, /*init=*/ true); fail("Did not catch expected IOException."); } catch (IOException ioe) { - assertTrue(ioe.getCause().getMessage().equals("Catch me if you can!")); + assertTrue(ioe.getCause().getMessage().equals("First exception")); + } + // repeat for getModifiedDocIds + try { + adAdaptor.clearLastCompleteGroupCatalog(); + pushGroupDefinitions(adAdaptor, configEntries, pusher, + /*fullPush=*/ false, /*init=*/ true); + fail("Did not catch expected IOException."); + } catch (IOException ioe) { + assertTrue(ioe.getCause().getMessage().equals("First exception")); + boolean reachedLastCall = false; + try { + /* second call fills the catalog */ + pushGroupDefinitions(adAdaptor, configEntries, pusher, + /*fullPush=*/ false, /*init=*/ false); + /* third call does the push without any exception */ + pushGroupDefinitions(adAdaptor, configEntries, pusher, + /*fullPush=*/ false, /*init=*/ false); + reachedLastCall = true; + /* last call redoes the push (and catches the second exception) */ + pushGroupDefinitions(adAdaptor, configEntries, pusher, + /*fullPush=*/ false, /*init=*/ false); + fail("Did not catch second expected IOException."); + } catch (IOException ioe2) { + assertTrue(ioe2.getCause().getMessage().equals("Second exception")); + assertTrue(reachedLastCall); + } } } - public static byte[] hexStringToByteArray(String s) { return AdServerTest.hexStringToByteArray(s); } @@ -962,7 +1048,7 @@ return strings; } - private Map<String, String> defaultConfig() { + private static Map<String, String> defaultConfig() { Map<String, String> strings = new HashMap<String, String>(); strings.put("gsa.hostname", "localhost"); strings.put("ad.servers", "server1"); @@ -1090,8 +1176,10 @@ public static void pushGroupDefinitions(AdAdaptor adaptor, Map<String, String> configEntries, final DocIdPusher pusher, - boolean fullPush) throws Exception { - initializeAdaptorConfig(adaptor, configEntries); + boolean fullPush, boolean init) throws Exception { + if (init) { + initializeAdaptorConfig(adaptor, configEntries); + } if (fullPush) { adaptor.getDocIds(pusher); } else { @@ -1205,12 +1293,23 @@ fail("Could not create LdapContext:" + e); } return new AdServer(host, ldapContext) { + private long highestCommittedUSN = 12345678; @Override void recreateLdapContext() { // leave ldapContext unchanged } + @Override + public long getHighestCommittedUSN() { + // always indicate new items available to sync + return ++highestCommittedUSN; + } }; } + @Override + void getModifiedDocIdsHelper(DocIdPusher pusher) + throws InterruptedException, IOException { + // do nothing + } }; /** Simple Fake of GroupCatalog that tracks calls to full/incremental crawl */ @@ -1224,15 +1323,24 @@ } @Override - void readEverythingFrom(AdServer server) throws InterruptedNamingException { - ranIncrementalCrawl = false; + void readEverythingFrom(AdServer server, boolean unused) + throws InterruptedNamingException { + try { + Thread.sleep(BRIEF_DELAY_IN_MILLISECONDS * 2); + } catch (InterruptedException ex) { + throw new InterruptedNamingException(ex.getMessage()); + } ranFullCrawl = true; } @Override Set<AdEntity> incrementalCrawl(AdServer server, long previousHighestUSN, long currentHighestUSN) throws InterruptedNamingException { - ranFullCrawl = false; + try { + Thread.sleep(BRIEF_DELAY_IN_MILLISECONDS * 2); + } catch (InterruptedException ex) { + throw new InterruptedNamingException(ex.getMessage()); + } ranIncrementalCrawl = true; return Collections.emptySet(); } @@ -1250,4 +1358,76 @@ return ranFullCrawl; } } + + /** An even "faker" version of AdAdaptor that only tests the mutex */ + public class MoreFakeAdaptor extends FakeAdaptor { + private boolean ranFullCrawl; + private boolean ranIncrementalCrawl; + + void resetCrawlFlags() { + ranFullCrawl = false; + ranIncrementalCrawl = false; + } + + public boolean ranIncrementalCrawl() { + return ranIncrementalCrawl; + } + + public boolean ranFullCrawl() { + return ranFullCrawl; + } + + @Override + void getModifiedDocIdsHelper(DocIdPusher pusher) + throws InterruptedException, IOException { + ranIncrementalCrawl = true; + } + + @Override + AdAdaptor.GroupCatalog makeFullCatalog() throws InterruptedException, + IOException { + try { + Thread.sleep(BRIEF_DELAY_IN_MILLISECONDS * 2); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + ranFullCrawl = true; + return new AdAdaptor.GroupCatalog(defaultLocalizedStringMap(), + "example.com", /*feedBuiltinGroups=*/ true); + } + }; + + /** generates a thread that invokes a (fake!) full crawl */ + private static class FullCrawlThread extends Thread { + FullCrawlThread (final AdAdaptor adAdaptor) { + super((new Runnable() { + @Override + public void run() { + try { + AccumulatingDocIdPusher pusher = new AccumulatingDocIdPusher(); + adAdaptor.getDocIds(pusher); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + })); + } + } + + /** generates a thread that invokes a (fake!) incremental crawl */ + private static class IncrementalCrawlThread extends Thread { + IncrementalCrawlThread (final AdAdaptor adAdaptor) { + super((new Runnable() { + @Override + public void run() { + try { + AccumulatingDocIdPusher pusher = new AccumulatingDocIdPusher(); + adAdaptor.getModifiedDocIds(pusher); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + })); + } + } }