send namespaced acls in http headers
diff --git a/src/com/google/enterprise/adaptor/Config.java b/src/com/google/enterprise/adaptor/Config.java
index 2fc1f9e..492cc1e 100644
--- a/src/com/google/enterprise/adaptor/Config.java
+++ b/src/com/google/enterprise/adaptor/Config.java
@@ -36,6 +36,11 @@
* <td><b>name</b></td><td><b>meaning</b></td>
* <tr><td> </td><td>adaptor.autoUnzip </td><td> expand zip files and send
* each file inside separatly. Defaults to false
+ * <tr><td> </td><td>adaptor.sendDocControlsHeader </td><td>use
+ * X-Gsa-Doc-Controls HTTP header with namespaced ACLs,
+ * lock attribute, crawl-once attribute.
+ * Otherwise ACLs are sent without namespace and as metadata.
+ * Also other noted items are not sent at all. Defaults to false
* <tr><td> </td><td>adaptor.fullListingSchedule </td><td> when to invoke
* {@link Adaptor#getDocIds Adaptor.getDocIds}, in cron format (minute,
* hour, day of month, month, day of week). Defaults to 0 3 * * *
@@ -207,6 +212,7 @@
addKey("transform.maxDocumentBytes", "1048576");
addKey("transform.required", "false");
addKey("journal.reducedMem", "true");
+ addKey("adaptor.sendDocControlsHeader", "false");
}
public Set<String> getAllKeys() {
@@ -394,6 +400,10 @@
return Boolean.parseBoolean(getValue("server.useCompression"));
}
+ public boolean sendDocControlsHeader() {
+ return Boolean.parseBoolean(getValue("adaptor.sendDocControlsHeader"));
+ }
+
/**
* Optional (default false): Adds no-recrawl bit with sent records in feed
* file. If connector handles updates and deletes then GSA does not have to
diff --git a/src/com/google/enterprise/adaptor/DocumentHandler.java b/src/com/google/enterprise/adaptor/DocumentHandler.java
index 910d871..a145e1f 100644
--- a/src/com/google/enterprise/adaptor/DocumentHandler.java
+++ b/src/com/google/enterprise/adaptor/DocumentHandler.java
@@ -16,10 +16,15 @@
import static java.util.Map.Entry;
+import com.google.common.annotations.VisibleForTesting;
+
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpsExchange;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -28,7 +33,6 @@
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
-import java.security.Principal;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -64,6 +68,7 @@
private final int transformMaxBytes;
private final boolean transformRequired;
private final boolean useCompression;
+ private final boolean sendDocControls;
/**
* {@code authnHandler} and {@code transform} may be {@code null}.
@@ -75,7 +80,7 @@
SessionManager<HttpExchange> sessionManager,
TransformPipeline transform, int transformMaxBytes,
boolean transformRequired, boolean useCompression,
- Watchdog watchdog) {
+ boolean sendDocControls, Watchdog watchdog) {
if (docIdDecoder == null || docIdEncoder == null || journal == null
|| adaptor == null || sessionManager == null || watchdog == null) {
throw new NullPointerException();
@@ -90,6 +95,7 @@
this.transformMaxBytes = transformMaxBytes;
this.transformRequired = transformRequired;
this.useCompression = useCompression;
+ this.sendDocControls = sendDocControls;
this.watchdog = watchdog;
initFullAccess(gsaHostname, fullAccessHosts);
@@ -123,7 +129,7 @@
private boolean requestIsFromFullyTrustedClient(HttpExchange ex) {
boolean trust;
if (ex instanceof HttpsExchange) {
- Principal principal;
+ java.security.Principal principal;
try {
principal = ((HttpsExchange) ex).getSSLSession().getPeerPrincipal();
} catch (SSLPeerUnverifiedException e) {
@@ -293,7 +299,8 @@
return (sb.length() == 0) ? "" : sb.substring(0, sb.length() - 1);
}
- static String formAclHeader(Acl acl, DocIdEncoder docIdEncoder) {
+ @VisibleForTesting
+ static String formUnqualifiedAclHeader(Acl acl, DocIdEncoder docIdEncoder) {
if (acl == null) {
return "";
}
@@ -301,7 +308,6 @@
acl = Acl.FAKE_EMPTY;
}
StringBuilder sb = new StringBuilder();
- // TODO: Use Principals instead of just names
for (UserPrincipal permitUser : acl.getPermitUsers()) {
String name = permitUser.getName();
percentEncodeMapEntryPair(sb, "google:aclusers", name);
@@ -329,6 +335,80 @@
return sb.substring(0, sb.length() - 1);
}
+ @VisibleForTesting
+ static String formNamespacedAclHeader(Acl acl, DocIdEncoder enc) {
+ if (null == acl) {
+ return "";
+ }
+ if (Acl.EMPTY.equals(acl)) {
+ acl = Acl.FAKE_EMPTY;
+ }
+ Map<String, Object> gsaAcl = makeGsaAclMap(acl, enc);
+ if (null == gsaAcl) {
+ return "";
+ } else {
+ return "acl=" + percentEncode("" + JSONObject.toJSONString(gsaAcl));
+ }
+ }
+
+ /** Gives null if there are no entires, no inherit-from and is LEAF_NODE. */
+ private static Map<String, Object> makeGsaAclMap(Acl acl, DocIdEncoder enc) {
+ boolean didPutSomething = false;
+ Map<String, Object> gsaAcl = new TreeMap<String, Object>();
+ List<Map<String, String>> gsaAclEntries = makeGsaAclEntries(acl);
+ if (!gsaAclEntries.isEmpty()) {
+ gsaAcl.put("entries", gsaAclEntries);
+ didPutSomething = true;
+ }
+ if (null != acl.getInheritFrom()) {
+ URI from = enc.encodeDocId(acl.getInheritFrom());
+ gsaAcl.put("inherit-from", "" + from);
+ didPutSomething = true;
+ }
+ if (acl.getInheritanceType() != Acl.InheritanceType.LEAF_NODE) {
+ String type = acl.getInheritanceType().getCommonForm();
+ gsaAcl.put("inheritance-type", "" + type);
+ didPutSomething = true;
+ }
+ if (didPutSomething) {
+ return gsaAcl;
+ } else {
+ return null;
+ }
+ }
+
+ private static List<Map<String, String>> makeGsaAclEntries(Acl acl) {
+ List<Map<String, String>> princ = new ArrayList<Map<String, String>>();
+ for (Principal p : acl.getPermitGroups()) {
+ princ.add(makeGsaAclEntry("permit", acl, p));
+ }
+ for (Principal p : acl.getDenyGroups()) {
+ princ.add(makeGsaAclEntry("deny", acl, p));
+ }
+ for (Principal p : acl.getPermitUsers()) {
+ princ.add(makeGsaAclEntry("permit", acl, p));
+ }
+ for (Principal p : acl.getDenyUsers()) {
+ princ.add(makeGsaAclEntry("deny", acl, p));
+ }
+ return princ;
+ }
+
+ private static Map<String, String> makeGsaAclEntry(String access,
+ Acl acl, Principal p) {
+ Map<String, String> gsaEntry = new TreeMap<String, String>();
+ gsaEntry.put("access", access);
+ gsaEntry.put("scope", p.isUser()? "user" : "group");
+ gsaEntry.put("name", p.getName());
+ if (!Principal.DEFAULT_NAMESPACE.equals(p.getNamespace())) {
+ gsaEntry.put("namespace", p.getNamespace());
+ }
+ if (!acl.isEverythingCaseSensitive()) {
+ gsaEntry.put("case-sensitivity-type", "" + acl.isEverythingCaseSensitive());
+ }
+ return gsaEntry;
+ }
+
/**
* Format the GSA-specific anchor header value for extra crawl-time anchors.
*/
@@ -480,6 +560,8 @@
private boolean noIndex;
private boolean noFollow;
private boolean noArchive;
+ private boolean crawlOnce;
+ private boolean lock;
public DocumentResponse(HttpExchange ex, DocId docId) {
this.ex = ex;
@@ -600,6 +682,22 @@
}
@Override
+ public void setCrawlOnce(boolean crawlOnlyOnce) {
+ if (state != State.SETUP) {
+ throw new IllegalStateException("Already responded");
+ }
+ this.crawlOnce = crawlOnlyOnce;
+ }
+
+ @Override
+ public void setLock(boolean docLock) {
+ if (state != State.SETUP) {
+ throw new IllegalStateException("Already responded");
+ }
+ this.lock = docLock;
+ }
+
+ @Override
public void setNoFollow(boolean noFollow) {
if (state != State.SETUP) {
throw new IllegalStateException("Already responded");
@@ -686,9 +784,20 @@
// Always specify metadata and ACLs, even when empty, to replace
// previous values.
ex.getResponseHeaders().add("X-Gsa-External-Metadata",
- formMetadataHeader(metadata));
- ex.getResponseHeaders().add("X-Gsa-External-Metadata",
- formAclHeader(acl, docIdEncoder));
+ formMetadataHeader(metadata));
+ if (!sendDocControls) {
+ ex.getResponseHeaders().add("X-Gsa-External-Metadata",
+ formUnqualifiedAclHeader(acl, docIdEncoder));
+ } else {
+ if (crawlOnce) {
+ ex.getResponseHeaders().add("X-Gsa-Doc-Controls", "crawl-once");
+ }
+ if (lock) {
+ ex.getResponseHeaders().add("X-Gsa-Doc-Controls", "lock");
+ }
+ ex.getResponseHeaders().add("X-Gsa-Doc-Controls",
+ formNamespacedAclHeader(acl, docIdEncoder));
+ }
if (!anchorUris.isEmpty()) {
ex.getResponseHeaders().add("X-Gsa-External-Anchor",
formAnchorHeader(anchorUris, anchorTexts));
diff --git a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
index d8b5843..24e9d68 100644
--- a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
+++ b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
@@ -253,7 +253,9 @@
createTransformPipeline(),
config.getTransformMaxDocumentBytes(),
config.isTransformRequired(),
- config.isServerToUseCompression(), watchdog)));
+ config.isServerToUseCompression(),
+ config.sendDocControlsHeader(),
+ watchdog)));
// Start communicating with other services. As a general rule, by this time
// we want all services we provide to be up and running. However, note that
diff --git a/src/com/google/enterprise/adaptor/Response.java b/src/com/google/enterprise/adaptor/Response.java
index edd13a9..7d61362 100644
--- a/src/com/google/enterprise/adaptor/Response.java
+++ b/src/com/google/enterprise/adaptor/Response.java
@@ -148,4 +148,20 @@
* link in search results
*/
public void setNoArchive(boolean noArchive);
+
+ /**
+ * Whether the GSA should crawl the document just one time and never again.
+ *
+ * @param crawlOnce {@code true} when doc should be crawled only once
+ */
+ public void setCrawlOnce(boolean crawlOnce);
+
+ /**
+ * Prevents the GSA from removing document from index. For example
+ * in cases where GSA is at license limit and is crawling more documents
+ * the GSA is not allowed to remove locked documents.
+ *
+ * @param lock {@code true} when doc is be kept in index
+ */
+ public void setLock(boolean lock);
}
diff --git a/src/com/google/enterprise/adaptor/WrapperAdaptor.java b/src/com/google/enterprise/adaptor/WrapperAdaptor.java
index fba8a82..8e54a27 100644
--- a/src/com/google/enterprise/adaptor/WrapperAdaptor.java
+++ b/src/com/google/enterprise/adaptor/WrapperAdaptor.java
@@ -162,6 +162,16 @@
public void setNoArchive(boolean noArchive) {
response.setNoArchive(noArchive);
}
+
+ @Override
+ public void setCrawlOnce(boolean crawlOnce) {
+ response.setCrawlOnce(crawlOnce);
+ }
+
+ @Override
+ public void setLock(boolean lock) {
+ response.setLock(lock);
+ }
}
/**
@@ -211,6 +221,8 @@
private boolean noIndex;
private boolean noFollow;
private boolean noArchive;
+ private boolean crawlOnce;
+ private boolean lock;
public GetContentsResponse(OutputStream os) {
this.os = os;
@@ -277,6 +289,16 @@
this.noArchive = noArchive;
}
+ @Override
+ public void setCrawlOnce(boolean crawlOnlyOnce) {
+ this.crawlOnce = crawlOnlyOnce;
+ }
+
+ @Override
+ public void setLock(boolean lockDoc) {
+ this.lock = lockDoc;
+ }
+
public String getContentType() {
return contentType;
}
diff --git a/test/com/google/enterprise/adaptor/DocumentHandlerTest.java b/test/com/google/enterprise/adaptor/DocumentHandlerTest.java
index dd3726d..e68f70d 100644
--- a/test/com/google/enterprise/adaptor/DocumentHandlerTest.java
+++ b/test/com/google/enterprise/adaptor/DocumentHandlerTest.java
@@ -54,7 +54,7 @@
thrown.expect(NullPointerException.class);
new DocumentHandler(null, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost", new String[0], handler,
- sessionManager, null, 0, false, false, new MockWatchdog());
+ sessionManager, null, 0, false, false, false, new MockWatchdog());
}
@Test
@@ -62,7 +62,7 @@
thrown.expect(NullPointerException.class);
new DocumentHandler(docIdCodec, null, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost", new String[0], handler,
- sessionManager, null, 0, false, false, new MockWatchdog());
+ sessionManager, null, 0, false, false, false, new MockWatchdog());
}
@Test
@@ -70,7 +70,7 @@
thrown.expect(NullPointerException.class);
new DocumentHandler(docIdCodec, docIdCodec, null, new PrivateMockAdaptor(),
"localhost", new String[0], handler, sessionManager, null, 0,
- false, false, new MockWatchdog());
+ false, false, false, new MockWatchdog());
}
@Test
@@ -79,7 +79,7 @@
new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), null,
"localhost", new String[0], handler, sessionManager, null, 0,
- false, false, new MockWatchdog());
+ false, false, false, new MockWatchdog());
}
@Test
@@ -88,7 +88,7 @@
new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), new PrivateMockAdaptor(),
"localhost", new String[0], handler, null, null, 0,
- false, false, new MockWatchdog());
+ false, false, false, new MockWatchdog());
}
@Test
@@ -97,7 +97,7 @@
new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), new PrivateMockAdaptor(),
"localhost", new String[0], handler, sessionManager, null, 0,
- false, false, null);
+ false, false, false, null);
}
@Test
@@ -127,7 +127,7 @@
DocumentHandler handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), new PrivateMockAdaptor(),
"localhost", new String[0], authnHandler, sessionManager, null, 0,
- false, false, new MockWatchdog());
+ false, false, false, new MockWatchdog());
handler.handle(ex);
assertEquals(1234, ex.getResponseCode());
}
@@ -215,7 +215,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[] {remoteIp, " "}, null, sessionManager, null, 0, false,
- false, new MockWatchdog());
+ false, false, new MockWatchdog());
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
assertArrayEquals(mockAdaptor.documentBytes, ex.getResponseBytes());
@@ -229,7 +229,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
MockHttpExchange httpEx = ex;
MockHttpsExchange ex = new MockHttpsExchange(httpEx, new MockSslSession(
new X500Principal("CN=localhost, OU=Unknown, O=Unknown, C=Unknown")));
@@ -246,7 +246,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
MockHttpExchange httpEx = ex;
MockHttpsExchange ex = new MockHttpsExchange(httpEx, new MockSslSession(
null));
@@ -260,7 +260,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
MockHttpExchange httpEx = ex;
MockHttpsExchange ex = new MockHttpsExchange(httpEx, new MockSslSession(
new KerberosPrincipal("someuser@not-domain")));
@@ -274,7 +274,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
MockHttpExchange httpEx = ex;
MockHttpsExchange ex = new MockHttpsExchange(httpEx, new MockSslSession(
new X500Principal("OU=Unknown, O=Unknown, C=Unknown")));
@@ -288,7 +288,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
MockHttpExchange httpEx = ex;
MockHttpsExchange ex = new MockHttpsExchange(httpEx, new MockSslSession(
new X500Principal("CN=nottrusted, OU=Unknown, O=Unknown, C=Unknown")));
@@ -303,7 +303,7 @@
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
new PrivateMockAdaptor(), remoteIp,
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
assertArrayEquals(mockAdaptor.documentBytes, ex.getResponseBytes());
@@ -353,7 +353,8 @@
};
handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), mockAdaptor, "localhost",
- new String[0], null, sessionManager, null, 0, false, false, watchdog);
+ new String[0], null, sessionManager, null, 0, false, false,
+ false, watchdog);
try {
thrown.expect(RuntimeException.class);
handler.handle(ex);
@@ -369,7 +370,8 @@
Watchdog watchdog = new Watchdog(100, executor);
handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), mockAdaptor, "localhost",
- new String[0], null, sessionManager, null, 0, false, false, watchdog);
+ new String[0], null, sessionManager, null, 0, false, false,
+ false, watchdog);
try {
handler.handle(ex);
} finally {
@@ -408,7 +410,7 @@
DocumentHandler handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), mockAdaptor, "localhost",
new String[] {remoteIp}, null, sessionManager, transform, 100,
- false, false, new MockWatchdog());
+ false, false, false, new MockWatchdog());
mockAdaptor.documentBytes = new byte[] {1, 2, 3};
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
@@ -449,7 +451,7 @@
DocumentHandler handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), mockAdaptor, "localhost",
new String[] {remoteIp}, null, sessionManager, transform, 3, false,
- false, new MockWatchdog());
+ false, false, new MockWatchdog());
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
assertArrayEquals(golden, ex.getResponseBytes());
@@ -480,7 +482,7 @@
DocumentHandler handler = new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), mockAdaptor, "localhost",
new String[] {remoteIp}, null, sessionManager, transform, 3, true,
- false, new MockWatchdog());
+ false, false, new MockWatchdog());
thrown.expect(IOException.class);
try {
handler.handle(ex);
@@ -844,7 +846,7 @@
DocumentHandler handler = new DocumentHandler(
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
adaptor, "localhost", new String[] {remoteIp, "someUnknownHost!@#$"},
- null, sessionManager, null, 0, false, false, new MockWatchdog());
+ null, sessionManager, null, 0, false, false, false, new MockWatchdog());
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
assertEquals(Arrays.asList("test=ing", "google%3Aaclinheritfrom="
@@ -876,7 +878,7 @@
DocumentHandler handler = new DocumentHandler(
docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
adaptor, "localhost", new String[0], null,
- sessionManager, null, 0, false, false, new MockWatchdog());
+ sessionManager, null, 0, false, false, false, new MockWatchdog());
handler.handle(ex);
assertEquals(200, ex.getResponseCode());
assertNull(ex.getResponseHeaders().getFirst("X-Gsa-External-Metadata"));
@@ -886,7 +888,7 @@
return new DocumentHandler(docIdCodec, docIdCodec,
new Journal(new MockTimeProvider()), adaptor, "localhost",
new String[0], null, sessionManager, null, 0, false, false,
- new MockWatchdog());
+ false, new MockWatchdog());
}
@Test
@@ -909,10 +911,18 @@
return new UserPrincipal(name);
}
+ private static UserPrincipal U(String name, String ns) {
+ return new UserPrincipal(name, ns);
+ }
+
private static GroupPrincipal G(String name) {
return new GroupPrincipal(name);
}
+ private static GroupPrincipal G(String name, String ns) {
+ return new GroupPrincipal(name, ns);
+ }
+
@Test
public void testFormAclHeader() {
final String golden
@@ -926,7 +936,7 @@
// by percentEncode to give %2520.
+ "google%3Aaclinheritfrom=http%3A%2F%2Flocalhost%2Fsome%2520docId,"
+ "google%3Aaclinheritancetype=parent-overrides";
- String result = DocumentHandler.formAclHeader(new Acl.Builder()
+ String result = DocumentHandler.formUnqualifiedAclHeader(new Acl.Builder()
.setPermitUsers(Arrays.asList(U("pu1"), U("uid=pu2,dc=com")))
.setPermitGroups(Arrays.asList(G("pg1"), G("gid=pg2,dc=com")))
.setDenyUsers(Arrays.asList(U("du1"), U("uid=du2,dc=com")))
@@ -938,14 +948,125 @@
}
@Test
- public void testFormAclHeaderNull() {
- assertEquals("", DocumentHandler.formAclHeader(null, new MockDocIdCodec()));
+ public void testFormUnqualifiedAclHeaderNull() {
+ assertEquals("", DocumentHandler
+ .formUnqualifiedAclHeader(null, new MockDocIdCodec()));
}
@Test
- public void testFormAclHeaderEmpty() {
+ public void testFormUnqualifiedAclHeaderEmpty() {
+ DocIdEncoder enc = new MockDocIdCodec();
assertEquals("google%3Aacldenyusers=google%3AfakeUserToPreventMissingAcl",
- DocumentHandler.formAclHeader(Acl.EMPTY, new MockDocIdCodec()));
+ DocumentHandler.formUnqualifiedAclHeader(Acl.EMPTY, enc));
+ }
+
+ @Test
+ public void testFormNamespacedAclHeaderNull() {
+ assertEquals("", DocumentHandler
+ .formNamespacedAclHeader(null, new MockDocIdCodec()));
+ }
+
+ @Test
+ public void testFormNamespacedAclHeaderEmpty() {
+ DocIdEncoder enc = new MockDocIdCodec();
+ String golden = "acl={\"entries\":[{"
+ + "\"access\":\"deny\""
+ + ","
+ + "\"name\":\"google:fakeUserToPreventMissingAcl\""
+ + ","
+ + "\"scope\":\"user\""
+ + "}]}";
+ String aclHeader = DocumentHandler.formNamespacedAclHeader(Acl.EMPTY, enc);
+ assertTrue(aclHeader.startsWith("acl="));
+ String aclValue = percentDecode(aclHeader.substring("acl=".length()));
+ assertEquals(golden, "acl=" + aclValue);
+ }
+
+ @Test
+ public void testFormNamespacedAclHeaderBusy() {
+ DocIdEncoder enc = new MockDocIdCodec();
+ String golden = "acl={"
+ + "\"entries\":["
+ + "{\"access\":\"permit\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"pg1@d.g\",\"scope\":\"group\"},"
+ + "{\"access\":\"permit\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"gid=pg2,dc=m\",\"namespace\":\"ns\","
+ + "\"scope\":\"group\"},"
+ + "{\"access\":\"deny\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"gid=dg2,dc=com\",\"scope\":\"group\"},"
+ + "{\"access\":\"deny\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"dg1@d.g\",\"namespace\":\"ns\","
+ + "\"scope\":\"group\"},"
+ + "{\"access\":\"permit\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"uid=pu2,dc=m\",\"scope\":\"user\"},"
+ + "{\"access\":\"permit\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"pu1@d.g\",\"namespace\":\"ns\","
+ + "\"scope\":\"user\"},"
+ + "{\"access\":\"deny\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"du1@d.g\",\"scope\":\"user\"},"
+ + "{\"access\":\"deny\",\"case-sensitivity-type\":\"false\","
+ + "\"name\":\"uid=du2,dc=com\",\"namespace\":\"ns\","
+ + "\"scope\":\"user\"}"
+ + "],"
+ + "\"inherit-from\":\"http:\\/\\/localhost\\/some%20docId\","
+ + "\"inheritance-type\":\"parent-overrides\""
+ + "}";
+
+ Acl busyAcl = new Acl.Builder()
+ .setPermitUsers(Arrays.asList(U("pu1@d.g", "ns"), U("uid=pu2,dc=m")))
+ .setPermitGroups(Arrays.asList(G("pg1@d.g"), G("gid=pg2,dc=m", "ns")))
+ .setDenyUsers(Arrays.asList(U("du1@d.g"), U("uid=du2,dc=com", "ns")))
+ .setDenyGroups(Arrays.asList(G("dg1@d.g", "ns"), G("gid=dg2,dc=com")))
+ .setInheritFrom(new DocId("some docId"))
+ .setInheritanceType(Acl.InheritanceType.PARENT_OVERRIDES)
+ .setEverythingCaseInsensitive()
+ .build();
+ String aclHeader = DocumentHandler.formNamespacedAclHeader(busyAcl, enc);
+ assertTrue(aclHeader.startsWith("acl="));
+ String aclValue = percentDecode(aclHeader.substring("acl=".length()));
+ assertEquals(golden, "acl=" + aclValue);
+ }
+
+ @Test
+ public void testLockSentIfTrue() throws Exception {
+ MockAdaptor adaptor = new MockAdaptor() {
+ @Override
+ public void getDocContent(Request request, Response response)
+ throws IOException {
+ response.setLock(true);
+ response.getOutputStream();
+ }
+ };
+ String remoteIp = ex.getRemoteAddress().getAddress().getHostAddress();
+ DocumentHandler handler = new DocumentHandler(
+ docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
+ adaptor, "localhost", new String[] {remoteIp, "someUnknownHost!@#$"},
+ null, sessionManager, null, 0, false, false, true, new MockWatchdog());
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("lock",
+ ex.getResponseHeaders().getFirst("X-Gsa-Doc-Controls"));
+ }
+
+ @Test
+ public void testCrawlOnceSentIfTrue() throws Exception {
+ MockAdaptor adaptor = new MockAdaptor() {
+ @Override
+ public void getDocContent(Request request, Response response)
+ throws IOException {
+ response.setCrawlOnce(true);
+ response.getOutputStream();
+ }
+ };
+ String remoteIp = ex.getRemoteAddress().getAddress().getHostAddress();
+ DocumentHandler handler = new DocumentHandler(
+ docIdCodec, docIdCodec, new Journal(new MockTimeProvider()),
+ adaptor, "localhost", new String[] {remoteIp, "someUnknownHost!@#$"},
+ null, sessionManager, null, 0, false, false, true, new MockWatchdog());
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("crawl-once",
+ ex.getResponseHeaders().getFirst("X-Gsa-Doc-Controls"));
}
@Test
@@ -974,6 +1095,75 @@
}
return Collections.unmodifiableMap(result);
}
- };
+ }
+ /** percentDecode method is defined in this test file */
+ @Test
+ public void testPercentDecoder() {
+ assertEquals("" + ((char)(10)), percentDecode("%0A"));
+ for (char c = 0; c < 256; c++) {
+ String encoded = DocumentHandler.percentEncode("" + c);
+ String decoded = percentDecode(encoded);
+ if (!decoded.equals("" + c)) {
+ int n = c;
+ throw new AssertionError("failed to encode/decode `" + n
+ + "' `" + c + "' `" + encoded + "' `" + decoded + "'");
+ }
+ }
+ String decoded = percentDecode("AaZz09-_.~"
+ + "%60%3D%2F%3F%2B%27%3B%5C%2F%22%21%40%23%24%25%5E%26"
+ + "%2A%28%29%5B%5D%7B%7D%C3%AB%01");
+ assertEquals("AaZz09-_.~`=/?+';\\/\"!@#$%^&*()[]{}ë\u0001", decoded);
+ }
+
+ private static int hexToInt(byte b) {
+ if (b >= '0' && b <= '9') {
+ return (byte)(b - '0');
+ } else if (b >= 'a' && b <= 'f') {
+ return (byte)(b - 'a') + 10;
+ } else if (b >= 'A' && b <= 'F') {
+ return (byte)(b - 'A') + 10;
+ } else {
+ throw new IllegalArgumentException("invalid hex byte: " + b);
+ }
+ }
+
+ private static String percentDecode(String encoded) {
+ try {
+ byte bytes[] = encoded.getBytes("ASCII");
+ ByteArrayOutputStream decoded = percentDecode(bytes);
+ return decoded.toString("UTF-8");
+ } catch (UnsupportedEncodingException uee) {
+ throw new RuntimeException(uee);
+ }
+ }
+
+ private static ByteArrayOutputStream percentDecode(byte encoded[]) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ int i = 0;
+ while (i < encoded.length) {
+ byte b = encoded[i];
+ if (b == '%') {
+ int iNeeded = i + 2; // need two more bytes
+ if (iNeeded >= encoded.length) {
+ throw new IllegalArgumentException("ends too early");
+ }
+ int highOrder = hexToInt(encoded[i + 1]);
+ int lowOrder = hexToInt(encoded[i + 2]);
+ int byteInInt = (highOrder << 4) | lowOrder;
+ b = (byte) byteInInt; // chops top bytes; could make negative
+ i += 3;
+ } else if ((b >= 'a' && b <= 'z')
+ || (b >= 'A' && b <= 'Z')
+ || (b >= '0' && b <= '9')
+ || b == '-' || b == '_' || b == '.' || b == '~') {
+ // pass through
+ i++;
+ } else {
+ throw new IllegalArgumentException("not percent encoded");
+ }
+ out.write(b);
+ }
+ return out;
+ }
}
diff --git a/test/com/google/enterprise/adaptor/prebuilt/CommandLineAdaptorTest.java b/test/com/google/enterprise/adaptor/prebuilt/CommandLineAdaptorTest.java
index 412382d..39aa96f 100644
--- a/test/com/google/enterprise/adaptor/prebuilt/CommandLineAdaptorTest.java
+++ b/test/com/google/enterprise/adaptor/prebuilt/CommandLineAdaptorTest.java
@@ -294,6 +294,8 @@
private boolean noIndex;
private boolean noFollow;
private boolean noArchive;
+ private boolean crawlOnce;
+ private boolean lock;
public ContentsResponseTestMock(OutputStream os) {
this.os = os;
@@ -361,6 +363,16 @@
this.noArchive = noArchive;
}
+ @Override
+ public void setCrawlOnce(boolean crawlOnlyOnce) {
+ this.crawlOnce = crawlOnlyOnce;
+ }
+
+ @Override
+ public void setLock(boolean lockDoc) {
+ this.lock = lockDoc;
+ }
+
public String getContentType() {
return contentType;
}
