Prevent hard-coded words from being indexed
For any words we add to generated HTML pages, we now tell the GSA to not
index them. They can still be used in snippets, but they won't
contribute to scoring.
In addition, we only use user-provided wording for HTML titles since I'm
not certain googleoff functions properly in the title (it may prevent
indexing, but it may still influence scoring) and because those titles
are always shown to the user in full (and so our titles now look better
in search results).
diff --git a/src/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriter.java b/src/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriter.java
index 00cf2df..dc66e0f 100644
--- a/src/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriter.java
+++ b/src/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriter.java
@@ -27,7 +27,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
-import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Locale;
@@ -118,13 +117,16 @@
}
this.docId = docId;
this.docUri = docIdEncoder.encodeDocId(docId);
- // TODO(ejona): Localize.
- String header = MessageFormat.format("{0} {1}",
- computeTypeHeaderLabel(type), computeLabel(label, docId));
+ String documentLabel = computeLabel(label, docId);
writer.write("<!DOCTYPE html>\n<html><head><title>");
- writer.write(escapeContent(header));
+ writer.write(escapeContent(documentLabel));
writer.write("</title></head><body><h1>");
- writer.write(escapeContent(header));
+ googleoffIndex();
+ // TODO(ejona): Localize.
+ writer.write(computeTypeHeaderLabel(type));
+ googleonIndex();
+ writer.write(" ");
+ writer.write(escapeContent(documentLabel));
writer.write("</h1>");
state = State.STARTED;
}
@@ -135,7 +137,9 @@
}
checkAndCloseSection();
writer.write("<p>");
+ googleoffIndex();
writer.write(escapeContent(computeTypeSectionLabel(type)));
+ googleonIndex();
writer.write("</p><ul>");
state = State.IN_SECTION;
}
diff --git a/test/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriterTest.java b/test/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriterTest.java
index 7ae96f6..79b49fc 100644
--- a/test/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriterTest.java
+++ b/test/com/google/enterprise/adaptor/sharepoint/HtmlResponseWriterTest.java
@@ -96,9 +96,9 @@
@Test
public void testBasicFlow() throws Exception {
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>Site s</title></head>"
- + "<body><h1>Site s</h1>"
- + "<p>Lists</p>"
+ + "<html><head><title>s</title></head>"
+ + "<body><h1><!--googleoff: index-->Site<!--googleon: index--> s</h1>"
+ + "<p><!--googleoff: index-->Lists<!--googleon: index--></p>"
+ "<ul><li><a href=\"s/l\">My List</a></li></ul>"
+ "</body></html>";
writer.start(new DocId("s"), ObjectType.SITE, null);
@@ -111,9 +111,9 @@
@Test
public void testOverflowToDocIdPusher() throws Exception {
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>Site s</title></head>"
- + "<body><h1>Site s</h1>"
- + "<p>Lists</p>"
+ + "<html><head><title>s</title></head>"
+ + "<body><h1><!--googleoff: index-->Site<!--googleon: index--> s</h1>"
+ + "<p><!--googleoff: index-->Lists<!--googleon: index--></p>"
+ "<ul><li><a href=\"s/l\">My List</a></li></ul>"
+ "</body></html>";
final List<DocIdPusher.Record> goldenRecords = Arrays.asList(
@@ -166,8 +166,9 @@
@Test
public void testFinishNoSections() throws Exception {
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>Site a</title></head>"
- + "<body><h1>Site a</h1></body></html>";
+ + "<html><head><title>a</title></head>"
+ + "<body><h1><!--googleoff: index-->Site<!--googleon: index--> a</h1>"
+ + "</body></html>";
writer.start(new DocId("a"), ObjectType.SITE, "");
writer.finish();
writer.close();
diff --git a/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java b/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
index be5b7df..e8b0d9e 100644
--- a/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
+++ b/test/com/google/enterprise/adaptor/sharepoint/SharePointAdaptorTest.java
@@ -451,9 +451,10 @@
adaptor.getDocContent(new GetContentsRequest(new DocId("")), response);
String responseString = new String(baos.toByteArray(), charset);
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>Virtual Server http://localhost:1/</title></head>"
- + "<body><h1>Virtual Server http://localhost:1/</h1>"
- + "<p>Sites</p><ul>"
+ + "<html><head><title>http://localhost:1/</title></head>"
+ + "<body><h1><!--googleoff: index-->Virtual Server"
+ + "<!--googleon: index--> http://localhost:1/</h1>"
+ + "<p><!--googleoff: index-->Sites<!--googleon: index--></p><ul>"
// These are relative URLs to DocIds that are URLs, and thus the "./"
// prefix is correct.
+ "<li><a href=\"./http://localhost:1\">localhost:1</a></li>"
@@ -544,12 +545,13 @@
adaptor.getDocContent(request, response);
String responseString = new String(baos.toByteArray(), charset);
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>Site chinese1</title></head>"
- + "<body><h1>Site chinese1</h1>"
- + "<p>Sites</p>"
+ + "<html><head><title>chinese1</title></head>"
+ + "<body><h1><!--googleoff: index-->Site<!--googleon: index-->"
+ + " chinese1</h1>"
+ + "<p><!--googleoff: index-->Sites<!--googleon: index--></p>"
+ "<ul><li><a href=\"SiteCollection/somesite\">"
+ "http://localhost:1/sites/SiteCollection/somesite</a></li></ul>"
- + "<p>Lists</p>"
+ + "<p><!--googleoff: index-->Lists<!--googleon: index--></p>"
+ "<ul><li><a href=\"SiteCollection/Lists/Announcements/"
+ "AllItems.aspx\">"
+ "/sites/SiteCollection/Lists/Announcements/AllItems.aspx</a></li>"
@@ -557,9 +559,9 @@
+ "AllItems.aspx\">"
+ "/sites/SiteCollection/Shared Documents/Forms/AllItems.aspx</a>"
+ "</li></ul>"
- + "<p>Folders</p>"
+ + "<p><!--googleoff: index-->Folders<!--googleon: index--></p>"
+ "<ul></ul>"
- + "<p>List Items</p>"
+ + "<p><!--googleoff: index-->List Items<!--googleon: index--></p>"
+ "<ul><li><a href=\"SiteCollection/default.aspx\">"
+ "default.aspx</a></li></ul>"
+ "</body></html>";
@@ -784,9 +786,10 @@
String responseString = new String(baos.toByteArray(), charset);
final String golden
= "<!DOCTYPE html>\n"
- + "<html><head><title>List Custom List</title></head>"
- + "<body><h1>List Custom List</h1>"
- + "<p>List Items</p>"
+ + "<html><head><title>Custom List</title></head>"
+ + "<body><h1><!--googleoff: index-->List<!--googleon: index-->"
+ + " Custom List</h1>"
+ + "<p><!--googleoff: index-->List Items<!--googleon: index--></p>"
+ "<ul>"
+ "<li><a href=\"3_.000\">Outside Folder</a></li>"
+ "<li><a href=\"Test%20Folder\">Test Folder</a></li>"
@@ -915,9 +918,10 @@
String responseString = new String(baos.toByteArray(), charset);
final String golden
= "<!DOCTYPE html>\n"
- + "<html><head><title>List Item Inside Folder</title></head>"
- + "<body><h1>List Item Inside Folder</h1>"
- + "<p>Attachments</p><ul>"
+ + "<html><head><title>Inside Folder</title></head>"
+ + "<body><h1><!--googleoff: index-->List Item<!--googleon: index-->"
+ + " Inside Folder</h1>"
+ + "<p><!--googleoff: index-->Attachments<!--googleon: index--></p><ul>"
+ "<li><a href=\"../Attachments/2/1046000.pdf\">1046000.pdf</a></li>"
+ "</ul>"
+ "<!--googleoff: index--><table style='border: none'>"
@@ -1121,8 +1125,9 @@
.getDocContent(request, response);
String responseString = new String(baos.toByteArray(), charset);
final String golden = "<!DOCTYPE html>\n"
- + "<html><head><title>List Item Inside Folder</title></head>"
- + "<body><h1>List Item Inside Folder</h1>"
+ + "<html><head><title>Inside Folder</title></head>"
+ + "<body><h1><!--googleoff: index-->List Item<!--googleon: index-->"
+ + " Inside Folder</h1>"
+ "<!--googleoff: index--><table style='border: none'>"
+ "<tr><td>Attachments</td><td>0</td></tr>"
+ "<tr><td>Author</td><td>System Account</td></tr>"
@@ -1268,9 +1273,10 @@
String responseString = new String(baos.toByteArray(), charset);
final String golden
= "<!DOCTYPE html>\n"
- + "<html><head><title>Folder Test Folder</title></head>"
- + "<body><h1>Folder Test Folder</h1>"
- + "<p>List Items</p>"
+ + "<html><head><title>Test Folder</title></head>"
+ + "<body><h1><!--googleoff: index-->Folder<!--googleon: index-->"
+ + " Test Folder</h1>"
+ + "<p><!--googleoff: index-->List Items<!--googleon: index--></p>"
+ "<ul>"
+ "<li><a href=\"Test%20Folder/2_.000\">Inside Folder</a></li>"
+ "<li><a href=\"Test%20Folder/testing\">testing</a></li>"
