commit a50fa6efd2de59eaea8e1a69c7e685624d05d554
author Eric Anderson <ejona@google.com> Tue Aug 27 13:58:05 2013 -0700
committer Eric Anderson <ejona@google.com> Tue Oct 22 10:24:14 2013 -0700
tree 60b04b3f12d249526a1ed769ebcfec5b3f0bd654
parent d3a39bbb08f594ba7f1f996e73e48e314fce54fc

Batch group definitions

This reduces our memory consumption

src/com/google/enterprise/adaptor/sharepoint/SharePointAdaptor.java

diff --git a/src/com/google/enterprise/adaptor/sharepoint/SharePointAdaptor.java b/src/com/google/enterprise/adaptor/sharepoint/SharePointAdaptor.java
index 5c57a39..11a467a 100644
--- a/src/com/google/enterprise/adaptor/sharepoint/SharePointAdaptor.java
+++ b/src/com/google/enterprise/adaptor/sharepoint/SharePointAdaptor.java
@@ -251,6 +251,7 @@
   private final Callable<ExecutorService> executorFactory;
   private ExecutorService executor;
   private boolean xmlValidation;
+  private int feedMaxUrls;
   private long maxIndexableSize;
   
   private ScheduledThreadPoolExecutor scheduledExecutor 
@@ -346,6 +347,7 @@
         config.getValue("sharepoint.password"));
     xmlValidation = Boolean.parseBoolean(
         config.getValue("sharepoint.xmlValidation"));
+    feedMaxUrls = Integer.parseInt(config.getValue("feed.maxUrls"));
     maxIndexableSize = Integer.parseInt(
         config.getValue("sharepoint.maxIndexableSize"));
     defaultNamespace = config.getValue("adaptor.namespace");
@@ -501,7 +503,16 @@
               siteString);
           continue;
         }
-        defs.putAll(siteAdaptor.computeMembersForGroups(site.getGroups()));
+        Map<GroupPrincipal, Collection<Principal>> siteDefs
+            = siteAdaptor.computeMembersForGroups(site.getGroups());
+        for (Map.Entry<GroupPrincipal, Collection<Principal>> me
+            : siteDefs.entrySet()) {
+          defs.put(me.getKey(), me.getValue());
+          if (defs.size() >= feedMaxUrls) {
+            pusher.pushGroupDefinitions(defs, false);
+            defs.clear();
+          }
+        }
       }
     }
     pusher.pushGroupDefinitions(defs, false);

test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java

diff --git a/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java b/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
index 3b325e3..770c7a7 100644
--- a/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
+++ b/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
@@ -1216,6 +1216,8 @@
       goldenGroups = Collections.unmodifiableMap(tmp);
     }
 
+    // Force a full batch of 2 and a final batch of 1.
+    config.overrideKey("feed.maxUrls", "2");
     adaptor = new SharePointAdaptor(MockSoapFactory.blank()
         .endpoint(AUTH_ENDPOINT, new MockAuthenticationSoap())
         .endpoint(VS_ENDPOINT, MockSiteData.blank()