Added HelloWorldConnector Example
diff --git a/src/com/google/enterprise/adaptor/Application.java b/src/com/google/enterprise/adaptor/Application.java
index 49d96b0..5392432 100644
--- a/src/com/google/enterprise/adaptor/Application.java
+++ b/src/com/google/enterprise/adaptor/Application.java
@@ -89,8 +89,7 @@
* manual shutdown. A shutdown hook is automatically installed that calls
* {@code stop()}.
*/
- public synchronized void start() throws IOException, InterruptedException,
- StartupException {
+ public synchronized void start() throws IOException, InterruptedException {
synchronized (this) {
daemonInit();
@@ -100,7 +99,16 @@
Runtime.getRuntime().addShutdownHook(shutdownHook);
}
- daemonStart();
+ boolean success = false;
+ try {
+ daemonStart();
+ success = true;
+ } finally {
+ if (!success) {
+ // Call daemonDestroy() and remove shutdown hook.
+ stop(0, TimeUnit.SECONDS);
+ }
+ }
}
/**
@@ -112,21 +120,41 @@
if (primaryServer != null) {
throw new IllegalStateException("Already started");
}
- primaryServer = createHttpServer(config);
- dashboardServer = createDashboardHttpServer(config);
- // Because once stopped the server can't be reused, we can't reuse its
- // bind()ed socket if we stop it. So although ideally we would start/stop in
- // daemonStart/daemonStop, we instead must do it in
- // daemonInit/daemonDestroy.
- primaryServer.start();
- dashboardServer.start();
+ boolean success = false;
+ try {
+ primaryServer = createHttpServer(config);
+ dashboardServer = createDashboardHttpServer(config);
+ // Because once stopped the server can't be reused, we can't reuse its
+ // bind()ed socket if we stop it. So although ideally we would start/stop
+ // in daemonStart/daemonStop, we instead must do it in
+ // daemonInit/daemonDestroy.
+ primaryServer.start();
+ dashboardServer.start();
+ success = true;
+ } finally {
+ if (!success) {
+ daemonDestroy(0, TimeUnit.SECONDS);
+ }
+ }
}
/**
* Really start. This must be called after a successful {@link #daemonInit}.
*/
- synchronized void daemonStart() throws IOException, InterruptedException,
- StartupException {
+ synchronized void daemonStart() throws IOException, InterruptedException {
+ boolean success = false;
+ try {
+ realDaemonStart();
+ success = true;
+ } finally {
+ if (!success) {
+ daemonStop(0, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ private synchronized void realDaemonStart() throws IOException,
+ InterruptedException {
AdaptorContext context = gsa.setup(primaryServer, dashboardServer, null);
long sleepDurationMillis = 8000;
@@ -150,8 +178,7 @@
} catch (InterruptedException ex) {
throw ex;
} catch (StartupException ex) {
- log.log(Level.WARNING, "Failed to initialize adaptor", ex);
- throw ex;
+ throw new RuntimeException("Failed to initialize adaptor", ex);
} catch (Exception ex) {
log.log(Level.WARNING, "Failed to initialize adaptor", ex);
if (shutdownSemaphore.tryAcquire(sleepDurationMillis,
@@ -483,34 +510,13 @@
// Setup providing content.
try {
- try {
- app.start();
- log.info("doc content serving started");
- } catch (StartupException e) {
- throw new RuntimeException("could not start serving", e);
- } catch (IOException e) {
- throw new RuntimeException("could not start serving", e);
- }
+ app.start();
+ log.info("doc content serving started");
} catch (InterruptedException e) {
- // Do not call stop - it has already been called.
+ Thread.currentThread().interrupt();
throw new RuntimeException("could not start serving", e);
- } catch (Throwable t) {
- // Abnormal termination. Make sure any services that may have been
- // started are shut down.
- try {
- app.stop(3, TimeUnit.SECONDS);
- } catch (Throwable ignored) {
- // It was a noble try. Just get out.
- }
- // Now rethrow.
- if (t instanceof RuntimeException) {
- throw (RuntimeException) t;
- } else if (t instanceof Error) {
- throw (Error) t;
- } else {
- // Very unlikely to happen.
- throw new RuntimeException(t);
- }
+ } catch (IOException e) {
+ throw new RuntimeException("could not start serving", e);
}
return app;
}
diff --git a/src/com/google/enterprise/adaptor/Config.java b/src/com/google/enterprise/adaptor/Config.java
index dd187ea..9d1cd53 100644
--- a/src/com/google/enterprise/adaptor/Config.java
+++ b/src/com/google/enterprise/adaptor/Config.java
@@ -75,13 +75,17 @@
* are already URLs and avoid them being inserted into adaptor
generated URLs. Defaults to false
* <tr><td> </td><td>feed.crawlImmediatelyBitEnabled </td><td> send bit telling
- * GSA to crawl immediately. Defaults to false
+ * GSA to crawl immediately.
+ * Defaults to not overriding adaptor's decision which is typically to send
+ * updates as crawl-immediately and let GSA schedule crawl of all other ids
+ * <tr><td> </td><td>feed.noRecrawlBitEnabled </td><td> send bit telling
+ * GSA to crawl your documents only once.
+ * Defaults to not overriding adaptor's decision which is typically to send
+ * all documents as recrawlable (equivalent to value of false)
* <tr><td> </td><td>feed.maxUrls </td><td> set max number of URLs included
* per feed file. Defaults to 5000
* <tr><td> </td><td>feed.name </td><td> source name used in feeds. Generated
* if not provided
- * <tr><td> </td><td>feed.noRecrawlBitEnabled </td><td> send bit telling
- * GSA to crawl your documents only once. Defaults to false
* <tr><td> </td><td>feed.archiveDirectory </td><td> specifies a directory in
* which all feeds sent to the GSA will be archived. Feeds that failed to
* be sent to the GSA will be tagged with "FAILED" in the file name.
@@ -235,8 +239,8 @@
return rawValue;
}
});
- addKey("feed.noRecrawlBitEnabled", "false");
- addKey("feed.crawlImmediatelyBitEnabled", "false");
+ addKey("feed.noRecrawlBitEnabled", "");
+ addKey("feed.crawlImmediatelyBitEnabled", "");
//addKey("feed.noFollowBitEnabled", "false");
addKey("feed.maxUrls", "5000");
addKey("adaptor.pushDocIdsOnStartup", "true");
@@ -442,21 +446,43 @@
return Boolean.parseBoolean(getValue("gsa.acceptsDocControlsHeader"));
}
- /**
- * Optional (default false): Adds no-recrawl bit with sent records in feed
- * file. If connector handles updates and deletes then GSA does not have to
- * recrawl periodically to notice that a document is changed or deleted.
- */
- boolean isFeedNoRecrawlBitEnabled() {
- return Boolean.getBoolean(getValue("feed.noRecrawlBitEnabled"));
+ static class OverridableBoolean {
+ final boolean isOverriden; // whether value is to be overriden
+ final boolean value; // the overriding value
+ private OverridableBoolean(boolean override) {
+ isOverriden = true;
+ value = override;
+ }
+ private OverridableBoolean() {
+ isOverriden = false;
+ value = false; // whatever
+ }
}
/**
- * Optional (default false): Adds crawl-immediately bit with sent records in
+ * Optional: Adds crawl-immediately bit with sent records in
* feed file. This bit makes the sent URL get crawl priority.
*/
- boolean isCrawlImmediatelyBitEnabled() {
- return Boolean.parseBoolean(getValue("feed.crawlImmediatelyBitEnabled"));
+ OverridableBoolean isCrawlImmediatelyBitEnabled() {
+ String provided = getValue("feed.crawlImmediatelyBitEnabled");
+ if ("".equals(provided.trim())) {
+ return new OverridableBoolean();
+ }
+ return new OverridableBoolean(Boolean.parseBoolean(provided));
+ }
+
+ /**
+ * Optional: Adds no-recrawl bit with sent records in feed
+ * file. If connector handles updates and deletes then GSA
+ * does not have to recrawl periodically to notice that a
+ * document is changed or deleted.
+ */
+ OverridableBoolean isFeedNoRecrawlBitEnabled() {
+ String provided = getValue("feed.noRecrawlBitEnabled");
+ if ("".equals(provided.trim())) {
+ return new OverridableBoolean();
+ }
+ return new OverridableBoolean(Boolean.parseBoolean(provided));
}
/**
@@ -720,6 +746,9 @@
public void validate() {
validate(config);
+ if ("".equals(getGsaHostname().trim())) {
+ throw new InvalidConfigurationException("gsa.hostname cannot be empty");
+ }
}
private void validate(Properties config) {
diff --git a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
index f5168a7..db77785 100644
--- a/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
+++ b/src/com/google/enterprise/adaptor/GsaCommunicationHandler.java
@@ -57,7 +57,6 @@
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.net.ssl.SSLException;
@@ -121,12 +120,6 @@
private KeyPair keyPair;
private AclTransform aclTransform;
- /**
- * Used to stop startup prematurely. When greater than 0, start() should abort
- * immediately because stop() is currently processing. This allows cancelling
- * new start() calls before stop() is done processing.
- */
- private final AtomicInteger shutdownCount = new AtomicInteger();
private ShutdownWaiter waiter;
private final List<Filter> commonFilters = Arrays.asList(new Filter[] {
new AbortImmediatelyFilter(),
@@ -215,7 +208,11 @@
aclTransform = createAclTransform();
GsaFeedFileMaker fileMaker = new GsaFeedFileMaker(docIdCodec, aclTransform,
config.isGsa614FeedWorkaroundEnabled(),
- config.isGsa70AuthMethodWorkaroundEnabled());
+ config.isGsa70AuthMethodWorkaroundEnabled(),
+ config.isCrawlImmediatelyBitEnabled().isOverriden,
+ config.isCrawlImmediatelyBitEnabled().value,
+ config.isFeedNoRecrawlBitEnabled().isOverriden,
+ config.isFeedNoRecrawlBitEnabled().value);
GsaFeedFileArchiver fileArchiver =
new GsaFeedFileArchiver(config.getFeedArchiveDirectory());
docIdSender = new DocIdSender(fileMaker, fileSender, fileArchiver, journal,
diff --git a/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java b/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
index 37a329d..82056fe 100644
--- a/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
+++ b/src/com/google/enterprise/adaptor/GsaFeedFileMaker.java
@@ -65,6 +65,10 @@
private final AclTransform aclTransform;
private final boolean separateClosingRecordTagWorkaround;
private final boolean useAuthMethodWorkaround;
+ private final boolean crawlImmediatelyIsOverriden;
+ private final boolean crawlImmediatelyOverrideValue;
+ private final boolean crawlOnceIsOverriden;
+ private final boolean crawlOnceOverrideValue;
public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform) {
this(encoder, aclTransform, false, false);
@@ -73,11 +77,26 @@
public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform,
boolean separateClosingRecordTagWorkaround,
boolean useAuthMethodWorkaround) {
+ this(encoder, aclTransform, separateClosingRecordTagWorkaround,
+ useAuthMethodWorkaround, false, false, false, false);
+ }
+
+ public GsaFeedFileMaker(DocIdEncoder encoder, AclTransform aclTransform,
+ boolean separateClosingRecordTagWorkaround,
+ boolean useAuthMethodWorkaround,
+ boolean overrideCrawlImmediately,
+ boolean crawlImmediately,
+ boolean overrideCrawlOnce,
+ boolean crawlOnce) {
this.idEncoder = encoder;
this.aclTransform = aclTransform;
this.separateClosingRecordTagWorkaround
= separateClosingRecordTagWorkaround;
this.useAuthMethodWorkaround = useAuthMethodWorkaround;
+ this.crawlImmediatelyIsOverriden = overrideCrawlImmediately;
+ this.crawlImmediatelyOverrideValue = crawlImmediately;
+ this.crawlOnceIsOverriden = overrideCrawlOnce;
+ this.crawlOnceOverrideValue = crawlOnce;
}
/** Adds header to document's root.
@@ -123,10 +142,15 @@
if (docRecord.isToBeLocked()) {
record.setAttribute("lock", "true");
}
- if (docRecord.isToBeCrawledImmediately()) {
+ if (crawlImmediatelyIsOverriden) {
+ record.setAttribute("crawl-immediately",
+ "" + crawlImmediatelyOverrideValue);
+ } else if (docRecord.isToBeCrawledImmediately()) {
record.setAttribute("crawl-immediately", "true");
}
- if (docRecord.isToBeCrawledOnce()) {
+ if (crawlOnceIsOverriden) {
+ record.setAttribute("crawl-once", "" + crawlOnceOverrideValue);
+ } else if (docRecord.isToBeCrawledOnce()) {
record.setAttribute("crawl-once", "true");
}
if (useAuthMethodWorkaround) {
diff --git a/src/com/google/enterprise/adaptor/InvalidConfigurationException.java b/src/com/google/enterprise/adaptor/InvalidConfigurationException.java
index c66798c..bccdf3e 100644
--- a/src/com/google/enterprise/adaptor/InvalidConfigurationException.java
+++ b/src/com/google/enterprise/adaptor/InvalidConfigurationException.java
@@ -21,7 +21,7 @@
*/
public class InvalidConfigurationException extends StartupException {
/**
- * Constructs a new InvalidConfigurationException with no message and no root
+ * Constructs a new InvalidConfigurationException with no message and no
* cause.
*/
public InvalidConfigurationException() {
@@ -29,8 +29,8 @@
}
/**
- * Constructs a InvalidConfigurationException with a supplied message
- * but no root cause.
+ * Constructs a InvalidConfigurationException with a supplied message but no
+ * cause.
*
* @param message the message. Can be retrieved by the {@link #getMessage()}
* method.
@@ -40,13 +40,25 @@
}
/**
- * Constructs a InvalidConfigurationException with message and root cause.
+ * Constructs a InvalidConfigurationException with message and cause.
*
* @param message the message. Can be retrieved by the {@link #getMessage()}
* method.
- * @param rootCause root failure cause
+ * @param cause failure cause
*/
- public InvalidConfigurationException(String message, Throwable rootCause) {
- super(message, rootCause);
+ public InvalidConfigurationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a InvalidConfigurationException with specified cause, copying
+ * its message if cause is non-{@code null}.
+ *
+ * @param message the message. Can be retrieved by the {@link #getMessage()}
+ * method.
+ * @param cause failure cause
+ */
+ public InvalidConfigurationException(Throwable cause) {
+ super(cause);
}
}
diff --git a/src/com/google/enterprise/adaptor/StartupException.java b/src/com/google/enterprise/adaptor/StartupException.java
index c8b5d3a..9464c46 100644
--- a/src/com/google/enterprise/adaptor/StartupException.java
+++ b/src/com/google/enterprise/adaptor/StartupException.java
@@ -17,21 +17,19 @@
/**
* Thrown for unrecoverable startup errors, such as fatal configuration
* errors or running on the wrong platform. StartupExceptions will bypass
- * the retry with back-off recovery logic and immediately terminate the
- * adaptor.
+ * the retry with back-off recovery logic of {@link Application} and immediately
+ * terminate the adaptor.
*/
public class StartupException extends RuntimeException {
/**
- * Constructs a new StartupException with no message and no root
- * cause.
+ * Constructs a new StartupException with no message and no cause.
*/
public StartupException() {
super();
}
/**
- * Constructs a StartupException with a supplied message but no root
- * cause.
+ * Constructs a StartupException with a supplied message but no cause.
*
* @param message the message. Can be retrieved by the {@link #getMessage()}
* method.
@@ -41,13 +39,23 @@
}
/**
- * Constructs a StartupException with message and root cause.
+ * Constructs a StartupException with message and cause.
*
* @param message the message. Can be retrieved by the {@link #getMessage()}
* method.
- * @param rootCause root failure cause
+ * @param cause failure cause
*/
- public StartupException(String message, Throwable rootCause) {
- super(message, rootCause);
+ public StartupException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a StartupException with specified cause, copying its message if
+ * cause is non-{@code null}.
+ *
+ * @param cause failure cause
+ */
+ public StartupException(Throwable cause) {
+ super(cause);
}
}
diff --git a/src/com/google/enterprise/adaptor/UnsupportedPlatformException.java b/src/com/google/enterprise/adaptor/UnsupportedPlatformException.java
index c004f36..3cd86d3 100644
--- a/src/com/google/enterprise/adaptor/UnsupportedPlatformException.java
+++ b/src/com/google/enterprise/adaptor/UnsupportedPlatformException.java
@@ -19,11 +19,10 @@
*/
public class UnsupportedPlatformException extends StartupException {
/**
- * Constructs a new UnsupportedPlatformException with a default message.
+ * Constructs a new UnsupportedPlatformException with no message.
*/
public UnsupportedPlatformException() {
- this(System.getProperty("os.name")
- + " is not a supported platform for this adaptor.");
+ super();
}
/**
@@ -35,4 +34,26 @@
public UnsupportedPlatformException(String message) {
super(message);
}
+
+ /**
+ * Constructs a UnsupportedPlatformException with a supplied message and
+ * cause.
+ *
+ * @param message the message. Can be retrieved by the {@link #getMessage()}
+ * method.
+ * @param cause failure cause
+ */
+ public UnsupportedPlatformException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a UnsupportedPlatformException with specified cause, copying its
+ * message if cause is non-{@code null}.
+ *
+ * @param cause failure cause
+ */
+ public UnsupportedPlatformException(Throwable cause) {
+ super(cause);
+ }
}
diff --git a/test/com/google/enterprise/adaptor/ApplicationTest.java b/test/com/google/enterprise/adaptor/ApplicationTest.java
index 12a9b8f..bce3236 100644
--- a/test/com/google/enterprise/adaptor/ApplicationTest.java
+++ b/test/com/google/enterprise/adaptor/ApplicationTest.java
@@ -276,10 +276,12 @@
@Test
public void testFailWithStartupException() throws Exception {
+ final StartupException startupException
+ = new StartupException("Unrecoverable error.");
class FailStartupAdaptor extends NullAdaptor {
@Override
public void init(AdaptorContext context) throws Exception {
- throw new StartupException("Unrecoverable error.");
+ throw startupException;
}
}
FailStartupAdaptor adaptor = new FailStartupAdaptor();
@@ -289,12 +291,13 @@
long startTime = System.nanoTime();
try {
app.start();
- fail("Expected a StartupException, but got none.");
- } catch (StartupException expected) {
+ fail("Expected a RuntimeException, but got none.");
+ } catch (RuntimeException expected) {
+ assertEquals(startupException, expected.getCause());
long duration = System.nanoTime() - startTime;
final long nanosInAMilli = 1000 * 1000;
if (duration > 1000 * nanosInAMilli) {
- fail("StartupException took a long time: " + duration);
+ fail("RuntimeException took a long time: " + duration);
}
}
}
diff --git a/test/com/google/enterprise/adaptor/ConfigTest.java b/test/com/google/enterprise/adaptor/ConfigTest.java
index 639d0f4..df4ca4c 100644
--- a/test/com/google/enterprise/adaptor/ConfigTest.java
+++ b/test/com/google/enterprise/adaptor/ConfigTest.java
@@ -49,6 +49,22 @@
}
@Test
+ public void testEmptyGsaHostname() {
+ // Requires gsa.hostname to be non-empty
+ config.setValue("gsa.hostname", "");
+ thrown.expect(InvalidConfigurationException.class);
+ config.validate();
+ }
+
+ @Test
+ public void testWhitespaceOnlyGsaHostname() {
+ // Requires gsa.hostname to be non-empty
+ config.setValue("gsa.hostname", " ");
+ thrown.expect(InvalidConfigurationException.class);
+ config.validate();
+ }
+
+ @Test
public void testAddDuplicateKeyWithValue() {
config.addKey("somekey", "value");
thrown.expect(IllegalStateException.class);
diff --git a/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java b/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
index 7eefa33..787448b 100644
--- a/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
+++ b/test/com/google/enterprise/adaptor/GsaFeedFileMakerTest.java
@@ -489,4 +489,174 @@
xml = xml.replaceAll("\r\n", "\n");
assertEquals(golden, xml);
}
+
+ @Test
+ public void testCrawlImmediatelyOverride() throws java.net.URISyntaxException {
+ GsaFeedFileMaker lclMeker = new GsaFeedFileMaker(encoder, aclTransform,
+ false, false,
+ /*override crawl-immediately?*/ true,
+ /*crawl-immediately value*/ false,
+ /*override crawl-once?*/ false,
+ /*crawl-once value*/ false);
+ String golden =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"
+ + "<!DOCTYPE gsafeed PUBLIC \"-//Google//DTD GSA Feeds//EN\" \"\">\n"
+ + "<gsafeed>\n"
+ + "<!--GSA EasyConnector-->\n"
+ + "<header>\n"
+ + "<datasource>t3sT</datasource>\n"
+ + "<feedtype>metadata-and-url</feedtype>\n"
+ + "</header>\n"
+ + "<group>\n"
+
+ // (1)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://f000nkey.net\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/E11\"/>\n"
+
+ // (2)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://yankee.doodle.com\""
+ + " last-modified=\"Thu, 01 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/elefenta\"/>\n"
+
+ // (3)
+ + "<record crawl-immediately=\"false\""
+ + " displayurl=\"http://google.com/news\""
+ + " last-modified=\"Fri, 02 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/gone\"/>\n"
+
+ // (4)
+ + "<record crawl-immediately=\"false\" crawl-once=\"true\""
+ + " lock=\"true\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/flagson\"/>\n"
+
+ // (5)
+ + "<record action=\"delete\" crawl-immediately=\"false\""
+ + " mimetype=\"text/plain\""
+ + " url=\"http://localhost/deleted\"/>\n"
+
+ + "</group>\n"
+ + "</gsafeed>\n";
+
+ ArrayList<DocIdPusher.Record> ids = new ArrayList<DocIdPusher.Record>();
+ DocIdPusher.Record.Builder attrBuilder
+ = new DocIdPusher.Record.Builder(new DocId("E11"));
+
+ // (1)
+ attrBuilder.setResultLink(new URI("http://f000nkey.net"));
+ ids.add(attrBuilder.build());
+
+ // (2)
+ attrBuilder.setResultLink(new URI("http://yankee.doodle.com"));
+ attrBuilder.setLastModified(new Date(0));
+ attrBuilder.setCrawlImmediately(true);
+ attrBuilder.setDocId(new DocId("elefenta"));
+ ids.add(attrBuilder.build());
+
+ // (3)
+ attrBuilder.setResultLink(new URI("http://google.com/news"));
+ attrBuilder.setLastModified(new Date(1000 * 60 * 60 * 24));
+ attrBuilder.setCrawlImmediately(false);
+ attrBuilder.setCrawlOnce(false);
+ attrBuilder.setDocId(new DocId("gone"));
+ ids.add(attrBuilder.build());
+
+ // (4)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("flagson"))
+ .setLock(true).setCrawlImmediately(true).setCrawlOnce(true).build());
+
+ // (5)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("deleted"))
+ .setDeleteFromIndex(true).build());
+
+ String xml = lclMeker.makeMetadataAndUrlXml("t3sT", ids);
+ xml = xml.replaceAll("\r\n", "\n");
+ assertEquals(golden, xml);
+ }
+
+ @Test
+ public void testCrawlOnceOverride() throws java.net.URISyntaxException {
+ GsaFeedFileMaker lclMeker = new GsaFeedFileMaker(encoder, aclTransform,
+ false, false,
+ /*override crawl-immediately?*/ false,
+ /*crawl-immediately value*/ false,
+ /*override crawl-once?*/ true,
+ /*crawl-once value*/ false);
+ String golden =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n"
+ + "<!DOCTYPE gsafeed PUBLIC \"-//Google//DTD GSA Feeds//EN\" \"\">\n"
+ + "<gsafeed>\n"
+ + "<!--GSA EasyConnector-->\n"
+ + "<header>\n"
+ + "<datasource>t3sT</datasource>\n"
+ + "<feedtype>metadata-and-url</feedtype>\n"
+ + "</header>\n"
+ + "<group>\n"
+
+ // (1)
+ + "<record crawl-once=\"false\""
+ + " displayurl=\"http://f000nkey.net\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/E11\"/>\n"
+
+ // (2)
+ + "<record crawl-immediately=\"true\" crawl-once=\"false\""
+ + " displayurl=\"http://yankee.doodle.com\""
+ + " last-modified=\"Thu, 01 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/elefenta\"/>\n"
+
+ // (3)
+ + "<record crawl-once=\"false\""
+ + " displayurl=\"http://google.com/news\""
+ + " last-modified=\"Fri, 02 Jan 1970 00:00:00 +0000\""
+ + " mimetype=\"text/plain\" url=\"http://localhost/gone\"/>\n"
+
+ // (4)
+ + "<record crawl-immediately=\"true\" crawl-once=\"false\""
+ + " lock=\"true\" mimetype=\"text/plain\""
+ + " url=\"http://localhost/flagson\"/>\n"
+
+ // (5)
+ + "<record action=\"delete\" crawl-once=\"false\""
+ + " mimetype=\"text/plain\""
+ + " url=\"http://localhost/deleted\"/>\n"
+
+ + "</group>\n"
+ + "</gsafeed>\n";
+
+ ArrayList<DocIdPusher.Record> ids = new ArrayList<DocIdPusher.Record>();
+ DocIdPusher.Record.Builder attrBuilder
+ = new DocIdPusher.Record.Builder(new DocId("E11"));
+
+ // (1)
+ attrBuilder.setResultLink(new URI("http://f000nkey.net"));
+ ids.add(attrBuilder.build());
+
+ // (2)
+ attrBuilder.setResultLink(new URI("http://yankee.doodle.com"));
+ attrBuilder.setLastModified(new Date(0));
+ attrBuilder.setCrawlImmediately(true);
+ attrBuilder.setDocId(new DocId("elefenta"));
+ ids.add(attrBuilder.build());
+
+ // (3)
+ attrBuilder.setResultLink(new URI("http://google.com/news"));
+ attrBuilder.setLastModified(new Date(1000 * 60 * 60 * 24));
+ attrBuilder.setCrawlImmediately(false);
+ attrBuilder.setCrawlOnce(false);
+ attrBuilder.setDocId(new DocId("gone"));
+ ids.add(attrBuilder.build());
+
+ // (4)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("flagson"))
+ .setLock(true).setCrawlImmediately(true).setCrawlOnce(true).build());
+
+ // (5)
+ ids.add(new DocIdPusher.Record.Builder(new DocId("deleted"))
+ .setDeleteFromIndex(true).build());
+
+ String xml = lclMeker.makeMetadataAndUrlXml("t3sT", ids);
+ xml = xml.replaceAll("\r\n", "\n");
+ assertEquals(golden, xml);
+ }
}
