Introduce incremental push logic into AdAdaptor
diff --git a/src/com/google/enterprise/adaptor/ad/AdAdaptor.java b/src/com/google/enterprise/adaptor/ad/AdAdaptor.java
index 816bb70..d318243 100644
--- a/src/com/google/enterprise/adaptor/ad/AdAdaptor.java
+++ b/src/com/google/enterprise/adaptor/ad/AdAdaptor.java
@@ -20,6 +20,7 @@
import com.google.enterprise.adaptor.Config;
import com.google.enterprise.adaptor.DocIdPusher;
import com.google.enterprise.adaptor.GroupPrincipal;
+import com.google.enterprise.adaptor.PollingIncrementalLister;
import com.google.enterprise.adaptor.Principal;
import com.google.enterprise.adaptor.Request;
import com.google.enterprise.adaptor.Response;
@@ -34,7 +35,8 @@
import javax.naming.InterruptedNamingException;
/** Adaptor for Active Directory. */
-public class AdAdaptor extends AbstractAdaptor {
+public class AdAdaptor extends AbstractAdaptor
+ implements PollingIncrementalLister {
private static final Logger log
= Logger.getLogger(AdAdaptor.class.getName());
private static final boolean CASE_SENSITIVITY = false;
@@ -68,7 +70,8 @@
defaultPassword = context.getConfig().getValue("ad.defaultPassword");
feedBuiltinGroups = Boolean.parseBoolean(
context.getConfig().getValue("ad.feedBuiltinGroups"));
-
+ // register for incremental pushes
+ context.setPollingIncrementalLister(this);
List<Map<String, String>> serverConfigs
= context.getConfig().getListOfConfigs("ad.servers");
servers.clear(); // in case init gets called again
@@ -136,15 +139,36 @@
@Override
public void getDocIds(DocIdPusher pusher) throws InterruptedException,
IOException {
+ readFromAllServers(pusher, false);
+ }
+
+ /**
+ * Attempts an incremntal push of all groups from all AdServers
+ *
+ * When a server cannot do an incremental push, it does a full push.
+ */
+ @Override
+ public void getModifiedDocIds(DocIdPusher pusher) throws InterruptedException,
+ IOException {
+ readFromAllServers(pusher, true);
+ }
+
+ public void readFromAllServers(DocIdPusher pusher,
+ boolean doIncrementalPushIfPossible)
+ throws InterruptedException, IOException {
// TODO(pjo): implement built in groups
GroupCatalog cumulativeCatalog = new GroupCatalog(localizedStrings,
namespace, feedBuiltinGroups);
for (AdServer server : servers) {
+ String previousServiceName = server.getDsServiceName();
+ String previousInvocationId = server.getInvocationID();
+ long previousHighestUSN = server.getHighestCommittedUSN();
server.initialize();
try {
GroupCatalog catalog = new GroupCatalog(localizedStrings, namespace,
feedBuiltinGroups);
- catalog.readFrom(server);
+ catalog.readFrom(server, doIncrementalPushIfPossible,
+ previousServiceName, previousInvocationId, previousHighestUSN);
cumulativeCatalog.add(catalog);
} catch (InterruptedNamingException ine) {
String host = server.getHostName();
@@ -237,7 +261,47 @@
this.domain.putAll(domain);
}
- void readFrom(AdServer server) throws InterruptedNamingException {
+ void readFrom(AdServer server, boolean doIncrementalPushIfPossible,
+ String previousServiceName, String previousInvocationId,
+ long previousHighestUSN) throws InterruptedNamingException {
+ if (!doIncrementalPushIfPossible) {
+ log.log(Level.INFO, "Starting full crawl.");
+ fullCrawl(server);
+ return;
+ }
+ // TODO(myk): Determine whether adaptors should include code to get/set
+ // last full sync time, and if exceeding some threshhold should force a
+ // full crawl.
+ String currentServiceName = server.getDsServiceName();
+ String currentInvocationId = server.getInvocationID();
+ long currentHighestUSN = server.getHighestCommittedUSN();
+ if (!currentServiceName.equals(previousServiceName)) {
+ log.log(Level.WARNING,
+ "Directory Controller changed from {0} to {1} -- performing full "
+ + "recrawl. Consider configuring AD server to connect directly to "
+ + "FQDN address of domain controller for partial updates support.",
+ new Object[]{previousServiceName, currentServiceName});
+ fullCrawl(server);
+ return;
+ }
+ if (!currentInvocationId.equals(previousInvocationId)) {
+ log.log(Level.WARNING,
+ "Directory Controller {0} has been restored from backup. "
+ + "Performing full recrawl.", currentServiceName);
+ fullCrawl(server);
+ return;
+ }
+ if (currentHighestUSN == previousHighestUSN) {
+ log.log(Level.INFO, "No updates on server {0} -- no crawl invoked.",
+ server);
+ return;
+ }
+ log.log(Level.INFO, "Starting incremental crawl.");
+ incrementalCrawl(server, previousHighestUSN, currentHighestUSN);
+ }
+
+ @VisibleForTesting
+ void fullCrawl(AdServer server) throws InterruptedNamingException {
// LDAP_MATCHING_RULE_BIT_AND = 1.2.840.113556.1.4.803
// and ADS_GROUP_TYPE_SECURITY_ENABLED = 2147483648.
entities = server.search("(|(&(objectClass=group)"
@@ -260,6 +324,37 @@
resolvePrimaryGroups();
}
+ @VisibleForTesting
+ void incrementalCrawl(AdServer server, long previousHighestUSN,
+ long currentHighestUSN) throws InterruptedNamingException {
+ // LDAP_MATCHING_RULE_BIT_AND = 1.2.840.113556.1.4.803
+ // and ADS_GROUP_TYPE_SECURITY_ENABLED = 2147483648.
+ Set<AdEntity> newEntities = server.search("(&(uSNChanged>"
+ + previousHighestUSN + ")(|(&(objectClass=group)"
+ + "(groupType:1.2.840.113556.1.4.803:=2147483648))"
+ + "(&(objectClass=user)(objectCategory=person))))",
+ /*deleted=*/ false,
+ new String[] { "uSNChanged", "sAMAccountName", "objectGUID;binary",
+ "objectSid;binary", "userPrincipalName", "primaryGroupId",
+ "member", "userAccountControl" });
+ // disabled groups handled later, in makeDefs()
+ log.log(Level.FINE, "received new {0} entities from server",
+ newEntities.size());
+ for (AdEntity e : newEntities) {
+ bySid.put(e.getSid(), e);
+ byDn.put(e.getDn(), e);
+ //TODO(myk): Handle tombstones / deleted items?
+ entities.add(e);
+ // TODO(pjo): Have AdServer put domain into AdEntity during search
+ domain.put(e, e.getSid().startsWith("S-1-5-32-") ?
+ localizedStrings.get("Builtin") : server.getnETBIOSName());
+ }
+ // TODO(myk): determine if these routines should be optimized to handle
+ // only the newly-discovered entities.
+ initializeMembers();
+ resolvePrimaryGroups();
+ }
+
private void initializeMembers() {
for (AdEntity entity : entities) {
if (entity.isGroup()) {
@@ -270,7 +365,7 @@
private void resolvePrimaryGroups() {
int nadds = 0;
- int missing_groups = 0;
+ int missingGroups = 0;
for (AdEntity e : entities) {
if (e.isGroup()) {
continue;
@@ -278,7 +373,7 @@
AdEntity user = e;
AdEntity primaryGroup = bySid.get(user.getPrimaryGroupSid());
if (primaryGroup == null) {
- missing_groups++;
+ missingGroups++;
log.log(Level.WARNING,
"Group {0} -- primary group for user {1} -- not found",
new Object[]{user.getPrimaryGroupSid(), user});
@@ -292,8 +387,8 @@
nadds++;
}
log.log(Level.FINE, "# primary groups: {0}", members.keySet().size());
- if (missing_groups > 0) {
- log.log(Level.FINE, "# missing primary groups: {0}", missing_groups);
+ if (missingGroups > 0) {
+ log.log(Level.FINE, "# missing primary groups: {0}", missingGroups);
}
log.log(Level.FINE, "# users added to all primary groups: {0}", nadds);
}
diff --git a/test/com/google/enterprise/adaptor/TestHelper.java b/test/com/google/enterprise/adaptor/TestHelper.java
index 0003d41..0c3b41f 100644
--- a/test/com/google/enterprise/adaptor/TestHelper.java
+++ b/test/com/google/enterprise/adaptor/TestHelper.java
@@ -34,6 +34,11 @@
public Config getConfig() {
return config;
}
+
+ @Override
+ public void setPollingIncrementalLister(PollingIncrementalLister lister) {
+ // do nothing
+ }
};
}
}
diff --git a/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java b/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java
index 6115d08..002e906 100644
--- a/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java
+++ b/test/com/google/enterprise/adaptor/ad/AdAdaptorTest.java
@@ -104,6 +104,8 @@
groupCatalog.add(golden);
groupCatalog.domain.clear();
assertFalse(golden.equals(groupCatalog));
+
+ assertFalse(golden.hashCode() == groupCatalog.hashCode());
}
@Test
@@ -131,7 +133,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
final AdEntity goldenEntity = new AdEntity("S-1-0-0",
"cn=name\\ under,DN_for_default_naming_context");
@@ -185,7 +188,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
AdEntity[] groupEntity = groupCatalog.entities.toArray(new AdEntity[0]);
final AdEntity goldenEntity = groupEntity[0];
@@ -247,7 +251,8 @@
adServer.initialize();
groupCatalog.bySid.put("S-1-5-32-users", userGroup);
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
final AdEntity goldenEntity = new AdEntity("S-1-5-32-544",
"cn=name\\ under,DN_for_default_naming_context", "users", "sam");
@@ -273,7 +278,8 @@
assertTrue(golden.equals(groupCatalog));
// make sure readFrom call is idempotent
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
assertTrue(golden.equals(groupCatalog));
}
@@ -300,7 +306,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
final AdEntity goldenEntity = new AdEntity("S-1-5-32-544",
"cn=name\\ under,DN_for_default_naming_context", "users", "sam");
@@ -324,7 +331,184 @@
assertTrue(golden.equals(groupCatalog));
// make sure readFrom call is idempotent
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
+ assertTrue(golden.equals(groupCatalog));
+ }
+
+ @Test
+ public void testGetModifiedDocIdsCallsReadFromAllServersCorrectly()
+ throws Exception {
+ FakeAdaptor adAdaptor = new FakeAdaptor() {
+ boolean attemptedIncrementalPush = false;
+ @Override
+ public void readFromAllServers(DocIdPusher pusher,
+ boolean doIncrementalPushIfPossible)
+ throws InterruptedException, IOException {
+ attemptedIncrementalPush = true;
+ }
+ @Override
+ public boolean equals(Object o) {
+ boolean results = attemptedIncrementalPush;
+ attemptedIncrementalPush = false;
+ return results;
+ }
+ };
+
+ assertFalse(adAdaptor.equals(null));
+ adAdaptor.getModifiedDocIds(null);
+ assertTrue(adAdaptor.equals(null));
+ assertFalse(adAdaptor.equals(null));
+ }
+
+ @Test
+ public void testFullCrawlVersusIncrementalCrawlFlow() throws Exception {
+ FakeAdaptor adAdaptor = new FakeAdaptor();
+ FakeCatalog groupCatalog = new FakeCatalog(defaultLocalizedStringMap(),
+ "example.com", false);
+ MockLdapContext ldapContext = defaultMockLdapContext();
+ AdServer adServer = new AdServer("localhost", ldapContext);
+ adServer.initialize();
+ // the following 3 lines initialize AdAdAptor.
+ AccumulatingDocIdPusher pusher = new AccumulatingDocIdPusher();
+ Map<String, String> configEntries = defaultConfig();
+ pushGroupDefinitions(adAdaptor, configEntries, pusher);
+
+ groupCatalog.resetCrawlFlags();
+ assertFalse(groupCatalog.ranFullCrawl);
+ assertFalse(groupCatalog.ranIncrementalCrawl);
+
+ groupCatalog.readFrom(adAdaptor.getServer(), false, "ds_service_name",
+ "0x0123456789abc", 12345678L);
+ assertTrue(groupCatalog.ranFullCrawl);
+ assertFalse(groupCatalog.ranIncrementalCrawl);
+
+ groupCatalog.resetCrawlFlags();
+ groupCatalog.readFrom(adAdaptor.getServer(), true, "other_ds_service_name",
+ "0x0123456789abc", 12345678L);
+ assertTrue(groupCatalog.ranFullCrawl);
+ assertFalse(groupCatalog.ranIncrementalCrawl);
+
+ groupCatalog.resetCrawlFlags();
+ groupCatalog.readFrom(adAdaptor.getServer(), true, "ds_service_name",
+ "otherInvocationId", 12345678L);
+ assertTrue(groupCatalog.ranFullCrawl);
+ assertFalse(groupCatalog.ranIncrementalCrawl);
+
+ groupCatalog.resetCrawlFlags();
+ groupCatalog.readFrom(adAdaptor.getServer(), true, "ds_service_name",
+ "0x0123456789abc", 12345678L); // last USN as previous run: no crawl
+ assertFalse(groupCatalog.ranFullCrawl);
+ assertFalse(groupCatalog.ranIncrementalCrawl);
+
+ // call Incremental call (only) when it's desired and the service name and
+ // invocation ID both match and the HighestUSN does not match.
+ groupCatalog.resetCrawlFlags();
+ groupCatalog.readFrom(adAdaptor.getServer(), true, "ds_service_name",
+ "0x0123456789abc", 12345677L);
+ assertFalse(groupCatalog.ranFullCrawl);
+ assertTrue(groupCatalog.ranIncrementalCrawl);
+ }
+
+ @Test
+ public void testGroupCatalogReadFromIncrementalCrawl() throws Exception {
+ Map<String, String> strings = defaultLocalizedStringMap();
+
+ AdAdaptor.GroupCatalog groupCatalog = new AdAdaptor.GroupCatalog(
+ strings, "example.com", /*feedBuiltinGroups=*/ false);
+ MockLdapContext ldapContext = defaultMockLdapContext();
+ // add a group
+ String filter = "(|(&(objectClass=group)"
+ + "(groupType:1.2.840.113556.1.4.803:=2147483648))"
+ + "(&(objectClass=user)(objectCategory=person)))";
+ String incrementalFilter = "(&(uSNChanged>12345677)" + filter + ")";
+ String searchDn = "DN_for_default_naming_context";
+ List<String> members = Arrays.asList("dn_for_user_1", "dn_for_user_2");
+ ldapContext.addSearchResult(filter, "cn", searchDn, "group_name")
+ .addSearchResult(filter, "objectSid;binary", searchDn,
+ hexStringToByteArray("010100000000000000000000")) // S-1-0-0
+ .addSearchResult(filter, "objectGUID;binary", searchDn,
+ hexStringToByteArray("000102030405060708090a0b0e"))
+ .addSearchResult(filter, "member", searchDn, members)
+ .addSearchResult(filter, "sAMAccountName", searchDn,
+ "name under");
+ ldapContext.addSearchResult(incrementalFilter, "cn", searchDn, "group_name")
+ .addSearchResult(incrementalFilter, "objectSid;binary", searchDn,
+ hexStringToByteArray("010100000000000000000000")) // S-1-0-0
+ .addSearchResult(incrementalFilter, "objectGUID;binary",
+ searchDn, hexStringToByteArray("000102030405060708090a0b0e"))
+ .addSearchResult(incrementalFilter, "member", searchDn, members)
+ .addSearchResult(incrementalFilter, "sAMAccountName", searchDn,
+ "name under");
+ // and a new user (under the incremental filter)
+ ldapContext.addSearchResult(incrementalFilter, "cn", searchDn, "admin_user")
+ .addSearchResult(incrementalFilter, "objectSid;binary", searchDn,
+ // S-1-5-32-544: local Admin. group
+ hexStringToByteArray("01020000000000052000000020020000"))
+ .addSearchResult(incrementalFilter, "objectGUID;binary", searchDn,
+ hexStringToByteArray("000102030405060708090a0b0e"))
+ .addSearchResult(incrementalFilter, "primaryGroupId", searchDn,
+ "users")
+ .addSearchResult(incrementalFilter, "sAMAccountName", searchDn,
+ "sam2");
+
+ AdServer adServer = new AdServer("localhost", ldapContext);
+ adServer.initialize();
+
+ // first, do a full crawl
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
+
+ // now do an incremental crawl
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ true,
+ "ds_service_name", "0x0123456789abc", 12345677L);
+
+ // extract incrementally-added user as one golden entity
+ Set<AdEntity> incrementalUserSet = adServer.search(incrementalFilter, false,
+ new String[] { "member", "objectSid;binary", "objectGUID;binary",
+ "primaryGroupId", "sAMAccountName" });
+ assertEquals(1, incrementalUserSet.size());
+ for (AdEntity ae : incrementalUserSet) {
+ assertFalse(ae.isGroup());
+ }
+ AdEntity[] searchedEntity = incrementalUserSet.toArray(new AdEntity[0]);
+ final AdEntity goldenUserEntity = searchedEntity[0];
+
+ // extract group from full crawl as the other golden entity
+ Set<AdEntity> fullyCrawledGroupSet = adServer.search(filter, false,
+ new String[] { "member", "objectSid;binary", "objectGUID;binary",
+ "primaryGroupId", "sAMAccountName" });
+ assertEquals(1, fullyCrawledGroupSet.size());
+ for (AdEntity ae : fullyCrawledGroupSet) {
+ assertTrue(ae.isGroup());
+ }
+ AdEntity[] groupEntity = fullyCrawledGroupSet.toArray(new AdEntity[0]);
+ final AdEntity goldenGroupEntity = new AdEntity("S-1-0-0",
+ "cn=name\\ under,DN_for_default_naming_context");
+ goldenGroupEntity.getMembers().addAll(members);
+
+ final Map<AdEntity, Set<String>> goldenMembers =
+ new HashMap<AdEntity, Set<String>>();
+ goldenMembers.put(goldenGroupEntity, goldenGroupEntity.getMembers());
+ final Map<String, AdEntity> goldenSid =
+ new HashMap<String, AdEntity>();
+ goldenSid.put(goldenGroupEntity.getSid(), goldenGroupEntity);
+ goldenSid.put(goldenUserEntity.getSid(), goldenUserEntity);
+ final Map<String, AdEntity> goldenDn =
+ new HashMap<String, AdEntity>();
+ goldenDn.put(goldenUserEntity.getDn(), goldenUserEntity);
+ final Map<AdEntity, String> goldenDomain = new HashMap<AdEntity, String>();
+ goldenDomain.put(goldenUserEntity, "BUILTIN");
+ goldenDomain.put(goldenGroupEntity, "GSA-CONNECTORS");
+
+ final AdAdaptor.GroupCatalog golden = new AdAdaptor.GroupCatalog(
+ strings, "example.com", /*feedBuiltinGroups=*/ true,
+ /*entities*/ Sets.newHashSet(goldenUserEntity, goldenGroupEntity),
+ /*members*/ goldenMembers,
+ /*bySid*/ goldenSid,
+ /*byDn*/ goldenDn,
+ /*domain*/ goldenDomain);
+
assertTrue(golden.equals(groupCatalog));
}
@@ -365,7 +549,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
// add two additional entities to test all branches of our method.
// first -- a user
@@ -430,7 +615,7 @@
assertTrue(golden.equals(groupCatalog));
- // make sure readFrom call is idempotent
+ // make sure resolveForeignSecurityPrincipals call is idempotent
groupCatalog.resolveForeignSecurityPrincipals();
assertTrue(golden.equals(groupCatalog));
}
@@ -447,7 +632,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
tweakGroupCatalogForMakeDefs(groupCatalog, adServer, false);
@@ -476,7 +662,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
tweakGroupCatalogForMakeDefs(groupCatalog, adServer, true);
@@ -515,7 +702,8 @@
AdServer adServer = new AdServer("localhost", ldapContext);
adServer.initialize();
- groupCatalog.readFrom(adServer);
+ groupCatalog.readFrom(adServer, /*doIncrementalPushIfPossible=*/ false,
+ "ds_service_name", "0x0123456789abc", 12345678L);
tweakGroupCatalogForMakeDefs(groupCatalog, adServer, false);
// now replace the parent group with a well-known one
@@ -976,6 +1164,7 @@
/** A version of AdAdaptor that uses only mock AdServers */
public class FakeAdaptor extends AdAdaptor {
+ private AdServer fakeServer = null;
@Override
AdServer newAdServer(Method method, String host, int port,
String principal, String passwd) {
@@ -985,7 +1174,41 @@
} catch (Exception e) {
fail("Could not create LdapContext:" + e);
}
- return new AdServer(host, ldapContext);
+ fakeServer = new AdServer(host, ldapContext);
+ return fakeServer;
+ }
+
+ AdServer getServer() {
+ return fakeServer;
+ }
+ };
+
+ /** Simple Fake of GroupCatalog that tracks calls to full/incremental crawl */
+ public class FakeCatalog extends AdAdaptor.GroupCatalog {
+ boolean ranFullCrawl;
+ boolean ranIncrementalCrawl;
+
+ public FakeCatalog(Map<String, String> localizedStrings, String namespace,
+ boolean feedBuiltinGroups) {
+ super(localizedStrings, namespace, feedBuiltinGroups);
+ }
+
+ @Override
+ void fullCrawl(AdServer server) throws InterruptedNamingException {
+ ranIncrementalCrawl = false;
+ ranFullCrawl = true;
+ }
+
+ @Override
+ void incrementalCrawl(AdServer server, long previousHighestUSN,
+ long currentHighestUSN) throws InterruptedNamingException {
+ ranFullCrawl = false;
+ ranIncrementalCrawl = true;
+ }
+
+ void resetCrawlFlags() {
+ ranFullCrawl = false;
+ ranIncrementalCrawl = false;
}
}
}
