allow override of crawl immediately
also allow override of crawl once
also actually use the two config options, in some way,
because currently they are not even used anywhere, it seems.
tests not included yet, want to see opinion on construct.
diff --git a/src/com/google/enterprise/adaptor/Config.java b/src/com/google/enterprise/adaptor/Config.java
index 56db897..9d1cd53 100644
--- a/src/com/google/enterprise/adaptor/Config.java
+++ b/src/com/google/enterprise/adaptor/Config.java
@@ -75,13 +75,17 @@
* are already URLs and avoid them being inserted into adaptor
generated URLs. Defaults to false
* <tr><td> </td><td>feed.crawlImmediatelyBitEnabled </td><td> send bit telling
- * GSA to crawl immediately. Defaults to false
+ * GSA to crawl immediately.
+ * Defaults to not overriding adaptor's decision which is typically to send
+ * updates as crawl-immediately and let GSA schedule crawl of all other ids
+ * <tr><td> </td><td>feed.noRecrawlBitEnabled </td><td> send bit telling
+ * GSA to crawl your documents only once.
+ * Defaults to not overriding adaptor's decision which is typically to send
+ * all documents as recrawlable (equivalent to value of false)
* <tr><td> </td><td>feed.maxUrls </td><td> set max number of URLs included
* per feed file. Defaults to 5000
* <tr><td> </td><td>feed.name </td><td> source name used in feeds. Generated
* if not provided
- * <tr><td> </td><td>feed.noRecrawlBitEnabled </td><td> send bit telling
- * GSA to crawl your documents only once. Defaults to false
* <tr><td> </td><td>feed.archiveDirectory </td><td> specifies a directory in
* which all feeds sent to the GSA will be archived. Feeds that failed to
* be sent to the GSA will be tagged with "FAILED" in the file name.
@@ -235,8 +239,8 @@
return rawValue;
}
});
- addKey("feed.noRecrawlBitEnabled", "false");
- addKey("feed.crawlImmediatelyBitEnabled", "false");
+ addKey("feed.noRecrawlBitEnabled", "");
+ addKey("feed.crawlImmediatelyBitEnabled", "");
//addKey("feed.noFollowBitEnabled", "false");
addKey("feed.maxUrls", "5000");
addKey("adaptor.pushDocIdsOnStartup", "true");
@@ -442,21 +446,43 @@
return Boolean.parseBoolean(getValue("gsa.acceptsDocControlsHeader"));
}
- /**
- * Optional (default false): Adds no-recrawl bit with sent records in feed
- * file. If connector handles updates and deletes then GSA does not have to
- * recrawl periodically to notice that a document is changed or deleted.
- */
- boolean isFeedNoRecrawlBitEnabled() {
- return Boolean.getBoolean(getValue("feed.noRecrawlBitEnabled"));
+ static class OverridableBoolean {
+ final boolean isOverriden; // whether value is to be overriden
+ final boolean value; // the overriding value
+ private OverridableBoolean(boolean override) {
+ isOverriden = true;
+ value = override;
+ }
+ private OverridableBoolean() {
+ isOverriden = false;
+ value = false; // whatever
+ }
}
/**
- * Optional (default false): Adds crawl-immediately bit with sent records in
+ * Optional: Adds crawl-immediately bit with sent records in
* feed file. This bit makes the sent URL get crawl priority.
*/
- boolean isCrawlImmediatelyBitEnabled() {
- return Boolean.parseBoolean(getValue("feed.crawlImmediatelyBitEnabled"));
+ OverridableBoolean isCrawlImmediatelyBitEnabled() {
+ String provided = getValue("feed.crawlImmediatelyBitEnabled");
+ if ("".equals(provided.trim())) {
+ return new OverridableBoolean();
+ }
+ return new OverridableBoolean(Boolean.parseBoolean(provided));
+ }
+
+ /**
+ * Optional: Adds no-recrawl bit with sent records in feed
+ * file. If connector handles updates and deletes then GSA
+ * does not have to recrawl periodically to notice that a
+ * document is changed or deleted.
+ */
+ OverridableBoolean isFeedNoRecrawlBitEnabled() {
+ String provided = getValue("feed.noRecrawlBitEnabled");
+ if ("".equals(provided.trim())) {
+ return new OverridableBoolean();
+ }
+ return new OverridableBoolean(Boolean.parseBoolean(provided));
}
/**
diff --git a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
index f5168a7..fa6fd0a 100644
--- a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
+++ b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
@@ -215,7 +215,11 @@
aclTransform = createAclTransform();
GsaFeedFileMaker fileMaker = new GsaFeedFileMaker(docIdCodec, aclTransform,
config.isGsa614FeedWorkaroundEnabled(),
- config.isGsa70AuthMethodWorkaroundEnabled());
+ config.isGsa70AuthMethodWorkaroundEnabled(),
+ config.isCrawlImmediatelyBitEnabled().isOverriden,
+ config.isCrawlImmediatelyBitEnabled().value,
+ config.isFeedNoRecrawlBitEnabled().isOverriden,
+ config.isFeedNoRecrawlBitEnabled().value);
GsaFeedFileArchiver fileArchiver =
new GsaFeedFileArchiver(config.getFeedArchiveDirectory());
docIdSender = new DocIdSender(fileMaker, fileSender, fileArchiver, journal,
diff --git a/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java b/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
index 37a329d..82056fe 100644
--- a/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
+++ b/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
@@ -65,6 +65,10 @@
private final AclTransform aclTransform;
private final boolean separateClosingRecordTagWorkaround;
private final boolean useAuthMethodWorkaround;
+ private final boolean crawlImmediatelyIsOverriden;
+ private final boolean crawlImmediatelyOverrideValue;
+ private final boolean crawlOnceIsOverriden;
+ private final boolean crawlOnceOverrideValue;
public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform) {
this(encoder, aclTransform, false, false);
@@ -73,11 +77,26 @@
public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform,
boolean separateClosingRecordTagWorkaround,
boolean useAuthMethodWorkaround) {
+ this(encoder, aclTransform, separateClosingRecordTagWorkaround,
+ useAuthMethodWorkaround, false, false, false, false);
+ }
+
+ public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform,
+ boolean separateClosingRecordTagWorkaround,
+ boolean useAuthMethodWorkaround,
+ boolean overrideCrawlImmediately,
+ boolean crawlImmediately,
+ boolean overrideCrawlOnce,
+ boolean crawlOnce) {
this.idEncoder = encoder;
this.aclTransform = aclTransform;
this.separateClosingRecordTagWorkaround
= separateClosingRecordTagWorkaround;
this.useAuthMethodWorkaround = useAuthMethodWorkaround;
+ this.crawlImmediatelyIsOverriden = overrideCrawlImmediately;
+ this.crawlImmediatelyOverrideValue = crawlImmediately;
+ this.crawlOnceIsOverriden = overrideCrawlOnce;
+ this.crawlOnceOverrideValue = crawlOnce;
}
/** Adds header to document's root.
@@ -123,10 +142,15 @@
if (docRecord.isToBeLocked()) {
record.setAttribute("lock", "true");
}
- if (docRecord.isToBeCrawledImmediately()) {
+ if (crawlImmediatelyIsOverriden) {
+ record.setAttribute("crawl-immediately",
+ "" + crawlImmediatelyOverrideValue);
+ } else if (docRecord.isToBeCrawledImmediately()) {
record.setAttribute("crawl-immediately", "true");
}
- if (docRecord.isToBeCrawledOnce()) {
+ if (crawlOnceIsOverriden) {
+ record.setAttribute("crawl-once", "" + crawlOnceOverrideValue);
+ } else if (docRecord.isToBeCrawledOnce()) {
record.setAttribute("crawl-once", "true");
}
if (useAuthMethodWorkaround) {
diff --git a/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java b/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
index 7eefa33..787448b 100644
--- a/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
+++ b/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
@@ -489,4 +489,174 @@
xml = xml.replaceAll("\r\n", "\n");
assertEquals(golden, xml);
}
+
+ @Test
+ public void testCrawlImmediatelyOverride() throws java.net.URISyntaxException {
+ GsaFeedFileMaker lclMeker = new GsaFeedFileMaker(encoder, aclTransform,
+ false, false,
+ /*override crawl-immediately?*/ true,
+ /*crawl-immediately value*/ false,
+ /*override crawl-once?*/ false,
+ /*crawl-once value*/ false);
+ String golden =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"
+ + "<!DOCTYPE gsafeed PUBLIC \"-//Google//DTD GSA Feeds//EN\" \"\">\n"
+ + "<gsafeed>\n"
+ + "<!--GSA EasyConnector-->\n"
+ + "<header>\n"
+ + "<datasource>t3sT</datasource>\n"
+ + "<feedtype>metadata-and-url</feedtype>\n"
+ + "</header>\n"
+ + "<group>\n"
+
+ // (1)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://f000nkey.net\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/E11\"/>\n"
+
+ // (2)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://yankee.doodle.com\""
+ + " last-modified=\"Thu, 01 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/elefenta\"/>\n"
+
+ // (3)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://google.com/news\""
+ + " last-modified=\"Fri, 02 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/gone\"/>\n"
+
+ // (4)
+ + "<record crawl-immediately=\"false\" crawl-once=\"true\""
+ + " lock=\"true\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/flagson\"/>\n"
+
+ // (5)
+ + "<record action=\"delete\" crawl-immediately=\"false\""
+ + " mimetype=\"text/plain\""
+ + " url=\"http://localhost/deleted\"/>\n"
+
+ + "</group>\n"
+ + "</gsafeed>\n";
+
+ ArrayList<DocIdPusher.Record> ids = new ArrayList<DocIdPusher.Record>();
+ DocIdPusher.Record.Builder attrBuilder
+ = new DocIdPusher.Record.Builder(new DocId("E11"));
+
+ // (1)
+ attrBuilder.setResultLink(new URI("http://f000nkey.net"));
+ ids.add(attrBuilder.build());
+
+ // (2)
+ attrBuilder.setResultLink(new URI("http://yankee.doodle.com"));
+ attrBuilder.setLastModified(new Date(0));
+ attrBuilder.setCrawlImmediately(true);
+ attrBuilder.setDocId(new DocId("elefenta"));
+ ids.add(attrBuilder.build());
+
+ // (3)
+ attrBuilder.setResultLink(new URI("http://google.com/news"));
+ attrBuilder.setLastModified(new Date(1000 * 60 * 60 * 24));
+ attrBuilder.setCrawlImmediately(false);
+ attrBuilder.setCrawlOnce(false);
+ attrBuilder.setDocId(new DocId("gone"));
+ ids.add(attrBuilder.build());
+
+ // (4)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("flagson"))
+ .setLock(true).setCrawlImmediately(true).setCrawlOnce(true).build());
+
+ // (5)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("deleted"))
+ .setDeleteFromIndex(true).build());
+
+ String xml = lclMeker.makeMetadataAndUrlXml("t3sT", ids);
+ xml = xml.replaceAll("\r\n", "\n");
+ assertEquals(golden, xml);
+ }
+
+ @Test
+ public void testCrawlOnceOverride() throws java.net.URISyntaxException {
+ GsaFeedFileMaker lclMeker = new GsaFeedFileMaker(encoder, aclTransform,
+ false, false,
+ /*override crawl-immediately?*/ false,
+ /*crawl-immediately value*/ false,
+ /*override crawl-once?*/ true,
+ /*crawl-once value*/ false);
+ String golden =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"
+ + "<!DOCTYPE gsafeed PUBLIC \"-//Google//DTD GSA Feeds//EN\" \"\">\n"
+ + "<gsafeed>\n"
+ + "<!--GSA EasyConnector-->\n"
+ + "<header>\n"
+ + "<datasource>t3sT</datasource>\n"
+ + "<feedtype>metadata-and-url</feedtype>\n"
+ + "</header>\n"
+ + "<group>\n"
+
+ // (1)
+ + "<record crawl-once=\"false\""
+ + " displayurl=\"http://f000nkey.net\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/E11\"/>\n"
+
+ // (2)
+ + "<record crawl-immediately=\"true\" crawl-once=\"false\""
+ + " displayurl=\"http://yankee.doodle.com\""
+ + " last-modified=\"Thu, 01 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/elefenta\"/>\n"
+
+ // (3)
+ + "<record crawl-once=\"false\""
+ + " displayurl=\"http://google.com/news\""
+ + " last-modified=\"Fri, 02 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/gone\"/>\n"
+
+ // (4)
+ + "<record crawl-immediately=\"true\" crawl-once=\"false\""
+ + " lock=\"true\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/flagson\"/>\n"
+
+ // (5)
+ + "<record action=\"delete\" crawl-once=\"false\""
+ + " mimetype=\"text/plain\""
+ + " url=\"http://localhost/deleted\"/>\n"
+
+ + "</group>\n"
+ + "</gsafeed>\n";
+
+ ArrayList<DocIdPusher.Record> ids = new ArrayList<DocIdPusher.Record>();
+ DocIdPusher.Record.Builder attrBuilder
+ = new DocIdPusher.Record.Builder(new DocId("E11"));
+
+ // (1)
+ attrBuilder.setResultLink(new URI("http://f000nkey.net"));
+ ids.add(attrBuilder.build());
+
+ // (2)
+ attrBuilder.setResultLink(new URI("http://yankee.doodle.com"));
+ attrBuilder.setLastModified(new Date(0));
+ attrBuilder.setCrawlImmediately(true);
+ attrBuilder.setDocId(new DocId("elefenta"));
+ ids.add(attrBuilder.build());
+
+ // (3)
+ attrBuilder.setResultLink(new URI("http://google.com/news"));
+ attrBuilder.setLastModified(new Date(1000 * 60 * 60 * 24));
+ attrBuilder.setCrawlImmediately(false);
+ attrBuilder.setCrawlOnce(false);
+ attrBuilder.setDocId(new DocId("gone"));
+ ids.add(attrBuilder.build());
+
+ // (4)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("flagson"))
+ .setLock(true).setCrawlImmediately(true).setCrawlOnce(true).build());
+
+ // (5)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("deleted"))
+ .setDeleteFromIndex(true).build());
+
+ String xml = lclMeker.makeMetadataAndUrlXml("t3sT", ids);
+ xml = xml.replaceAll("\r\n", "\n");
+ assertEquals(golden, xml);
+ }
}
