Merge branch 'master' of https://code.google.com/p/plexi
diff --git a/resources/com/google/enterprise/adaptor/TranslationStrings.properties b/resources/com/google/enterprise/adaptor/TranslationStrings.properties
index 38b4c82..47a9e43 100644
--- a/resources/com/google/enterprise/adaptor/TranslationStrings.properties
+++ b/resources/com/google/enterprise/adaptor/TranslationStrings.properties
@@ -38,6 +38,8 @@
HTTP header.
# The server encountered an error of some sort while processing the request.
HTTP_INTERNAL_ERROR=Error 500: Internal Server Error
+# What to display as the configuration filename when no file was used.
+STATS_CONFIG_NONE=none
# Shown when adaptor version is not known.
STATS_VERSION_UNKNOWN=Unknown
# The name of a status that monitors whether the GSA has made a request
@@ -55,5 +57,3 @@
STATUS_FEED=Feed Pushing
# The last data feed was interrupted before completion.
STATUS_FEED_INTERRUPTED=Push was interrupted
-# What to display as the configure filename when none exists.
-STATUS_NONE=none
diff --git a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
index 6547b12..e97acb4 100644
--- a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
+++ b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
@@ -15,6 +15,7 @@
package com.google.enterprise.adaptor;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
@@ -24,8 +25,9 @@
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
-import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -48,7 +50,7 @@
private String feedName;
/** Used to obtain statistics for one file in the .zip */
- private StatRpcMethod statRpcMethod;
+ private final StatRpcMethod statRpcMethod;
/** Used to specify the top-level directory where logs are kept */
private final File logsDir;
@@ -217,9 +219,6 @@
*/
private void dumpStats(ZipOutputStream zos) throws IOException {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(zos, "UTF-8"));
- String newline = "\n"; // so our support folks always see the same results
- // LinkedHashMap maintains put() -> get() order of elements.
- LinkedHashMap<String, String> stats = new LinkedHashMap<String, String>();
if (statRpcMethod == null) {
return; // don't generate empty stats file
}
@@ -231,53 +230,43 @@
if (null != map.get("versionStats")) {
@SuppressWarnings("unchecked")
Map<String, Object> vMap = (Map<String, Object>) map.get("versionStats");
- // TODO(myk): determine if these strings can be shared with the CSS code
- // that displays these in the dashboard
- stats.put("JVM version", vMap.get("versionJvm").toString());
- stats.put("Adaptor library version",
- vMap.get("versionAdaptorLibrary").toString());
- stats.put("Adaptor type", vMap.get("typeAdaptor").toString());
- stats.put("Adaptor version", vMap.get("versionAdaptor").toString());
- stats.put("Configuration file", vMap.get("configFileName").toString());
- stats.put("current directory", vMap.get("cwd").toString());
- prettyPrintMap(writer, stats);
- stats.clear();
+ prettyPrintMap(writer, vMap);
}
if (null != map.get("simpleStats")) {
@SuppressWarnings("unchecked")
Map<String, Object> sMap = (Map<String, Object>) map.get("simpleStats");
- stats.put("Program started at",
- prettyDate(sMap.get("whenStarted"), "Unknown"));
- stats.put("Last successful full push start",
- prettyDate(sMap.get("lastSuccessfulFullPushStart"), "None yet"));
- stats.put("Last successful full push end",
- prettyDate(sMap.get("lastSuccessfulFullPushEnd"), "None yet"));
- stats.put("Current full push",
- prettyDate(sMap.get("currentFullPushStart"), "None in progress"));
- stats.put("Last successful incremental push start",
- prettyDate(sMap.get("lastSuccessfulIncrementalPushStart"),
- "None yet"));
- stats.put("Last successful incremental push end",
- prettyDate(sMap.get("lastSuccessfulIncrementalPushEnd"), "None yet"));
- stats.put("Current incremental push",
- prettyDate(sMap.get("currentIncrementalPushStart"),
- "None in progress"));
- stats.put("Total document ids pushed",
- sMap.get("numTotalDocIdsPushed").toString());
- stats.put("Unique document ids pushed",
- sMap.get("numUniqueDocIdsPushed").toString());
- stats.put("GSA document requests",
- sMap.get("numTotalGsaRequests").toString());
- stats.put("GSA Unique document requests",
- sMap.get("numUniqueGsaRequests").toString());
- stats.put("Non-GSA document requests",
- sMap.get("numTotalNonGsaRequests").toString());
- stats.put("Non-GSA Unique document requests",
- sMap.get("numUniqueNonGsaRequests").toString());
- stats.put("Time resolution", sMap.get("timeResolution") + " ms");
- prettyPrintMap(writer, stats);
- stats.clear();
+ Set<String> expectedDateAttrs = ImmutableSet.of("whenStarted",
+ "lastSuccessfulFullPushStart", "lastSuccessfulFullPushEnd",
+ "currentFullPushStart", "lastSuccessfulIncremementalPushStart",
+ "lastSuccessfulIncremementalPushEnd", "currentIncrementalPushEnd");
+ DateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss Z");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
+ for (String key : expectedDateAttrs) {
+ if (!sMap.containsKey(key)) {
+ log.log(Level.INFO,
+ "Did not find expected key \"{0}\" in simpleStats", key);
+ continue;
+ }
+ Object value = sMap.get(key);
+ if (value == null) {
+ continue;
+ }
+ if (!(value instanceof Number)) {
+ log.log(Level.INFO,
+ "Key \"{0}\" contained non-date value \"{1}\" in simpleStats",
+ new Object[] {key, value});
+ continue;
+ }
+ long date = ((Number) value).longValue();
+ if (date <= 0) {
+ continue;
+ }
+ // It's a number > 0 -- assume it's a date.
+ // Replace value in map with a formatted date.
+ sMap.put(key, dateFormat.format(new Date(date)));
+ }
+ prettyPrintMap(writer, sMap);
}
writer.flush();
@@ -287,7 +276,7 @@
/**
* Pretty-prints a map
*/
- void prettyPrintMap(PrintWriter writer, Map<String, String> map) {
+ void prettyPrintMap(PrintWriter writer, Map<String, ?> map) {
int maxKeyLength = 0;
for (String key : map.keySet()) {
@@ -297,28 +286,13 @@
}
String outputFormat = "%-" + (maxKeyLength + 1) + "s= %s%n";
- for (Map.Entry<String, String> me : map.entrySet()) {
+ for (Map.Entry<String, ?> me : map.entrySet()) {
writer.format(outputFormat, me.getKey(),
- (me.getValue() == null ? "[null]" : me.getValue()));
+ (me.getValue() == null ? "[null]" : me.getValue().toString()));
}
writer.format("%n");
}
- private String prettyDate(Object date, String defaultText) {
- if (null == date) {
- return defaultText;
- }
- try {
- long value = new Long(date.toString());
- if (value == 0) {
- return defaultText;
- }
- return new Date(value).toString();
- } catch (NumberFormatException e) {
- return defaultText;
- }
- }
-
/**
* Method gets overriden in test class to avoid using "real" IO.
*/
diff --git a/src/com/google/enterprise/adaptor/StatRpcMethod.java b/src/com/google/enterprise/adaptor/StatRpcMethod.java
index fe53810..ae1e519 100644
--- a/src/com/google/enterprise/adaptor/StatRpcMethod.java
+++ b/src/com/google/enterprise/adaptor/StatRpcMethod.java
@@ -22,6 +22,8 @@
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Provides performance data when responding to
@@ -33,6 +35,8 @@
private Journal journal;
private boolean isAdaptorIncremental;
private File configFile;
+ private static final Logger log
+ = Logger.getLogger(StatRpcMethod.class.getName());
public StatRpcMethod(Journal journal, Adaptor adaptor,
boolean isAdaptorIncremental, File configFile) {
@@ -154,11 +158,13 @@
canonicalConfigFile = configFile.getCanonicalFile();
}
} catch (IOException e) {
- // ignore error; treat file as null
+ log.log(Level.WARNING, "Could not determine file location for \""
+ + configFile + "\"", e);
+ // treat file as if it were not specified -- leave it null below
}
return canonicalConfigFile == null ?
- Translation.STATUS_NONE.toString(locale) :
+ Translation.STATS_CONFIG_NONE.toString(locale) :
canonicalConfigFile.toString();
}
}
diff --git a/src/com/google/enterprise/adaptor/Translation.java b/src/com/google/enterprise/adaptor/Translation.java
index 06c0e9a..0ac47db 100644
--- a/src/com/google/enterprise/adaptor/Translation.java
+++ b/src/com/google/enterprise/adaptor/Translation.java
@@ -38,6 +38,7 @@
HTTP_BAD_METHOD,
HTTP_CONFLICT_INVALID_HEADER,
HTTP_INTERNAL_ERROR,
+ STATS_CONFIG_NONE,
STATS_VERSION_UNKNOWN,
STATUS_CRAWLING,
STATUS_CRAWLING_NO_ACCESSES_IN_PAST_DAY,
@@ -45,7 +46,6 @@
STATUS_ERROR_RATE_RATE,
STATUS_FEED,
STATUS_FEED_INTERRUPTED,
- STATUS_NONE,
;
/**
diff --git a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
index be693ed..e8514ac 100644
--- a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
+++ b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
@@ -16,6 +16,7 @@
import static org.junit.Assert.*;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -25,31 +26,45 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.Charset;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
+import java.util.TreeMap;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+
/**
* Tests for {@link DownloadDumpHandler}.
*/
public class DownloadDumpHandlerTest {
- private Config config;
- private DownloadDumpHandler handler;
+ private final Config config = new Config();
+ private DownloadDumpHandler handler =
+ new ModifiedDownloadDumpHandler(config, "adaptor");
private final String pathPrefix = "/";
- private MockHttpContext httpContext;
- private MockHttpExchange ex;
+ private final MockHttpContext httpContext =
+ new MockHttpContext(handler, pathPrefix);
+ private MockHttpExchange ex = createExchange("");
+ private TimeZone previousTimeZone;
@Rule
public ExpectedException thrown = ExpectedException.none();
@Before
public void setUp() {
- config = new Config();
// problems arise if gsa.version is left unset
config.setValue("gsa.version", "7.0.14-114");
- handler = new ModifiedDownloadDumpHandler(config, "adaptor");
- httpContext = new MockHttpContext(handler, pathPrefix);
- ex = createExchange("");
+ previousTimeZone = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("PST"));
+ }
+
+ @After
+ public void tearDown() {
+ TimeZone.setDefault(previousTimeZone);
}
@Test
@@ -87,8 +102,6 @@
@Test
public void testLogFilesWithCannedLogsDir() throws Exception {
// set up File using MockFile
- TimeZone previousTimeZone = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("PST"));
MockFile mockLogsDir = new MockFile("parentDir").setChildren(new File[] {
new MockFile("log1.log").setFileContents("Log file 1"),
new MockFile("log1.log.lck").setFileContents("skipped lock file"),
@@ -98,39 +111,149 @@
})});
MockTimeProvider timeProvider = new MockTimeProvider();
timeProvider.time = 1383763470000L; // November 6, 2013 @ 10:44:30
- try {
- handler = new ModifiedDownloadDumpHandler(config, "adaptor", mockLogsDir,
- timeProvider);
- handler.handle(ex);
- assertEquals(200, ex.getResponseCode());
- assertEquals("application/zip",
- ex.getResponseHeaders().getFirst("Content-Type"));
- assertEquals("attachment; filename=\"adaptor-20131106.zip\"",
- ex.getResponseHeaders().getFirst("Content-Disposition"));
- // extract the zip contents and just count the number of entries
- int entries = countZipEntries(ex.getResponseBytes());
- assertEquals(4, entries); /* 2 log files + thread dump + config */
- } finally {
- TimeZone.setDefault(previousTimeZone);
- }
+ StatRpcMethod statRpcMethod = new MockStatRpcMethod(
+ new MockJournal(new MockTimeProvider()),
+ new MockAdaptor(),
+ /*isAdaptorIncremental=*/ false,
+ /*configFile=*/ new MockFile("no-such-dir").setExists(false));
+ handler = new ModifiedDownloadDumpHandler(config, "adaptor",
+ statRpcMethod, mockLogsDir, timeProvider);
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("application/zip",
+ ex.getResponseHeaders().getFirst("Content-Type"));
+ assertEquals("attachment; filename=\"adaptor-20131106.zip\"",
+ ex.getResponseHeaders().getFirst("Content-Disposition"));
+ // extract the zip contents and just count the number of entries
+ byte[] zipContents = ex.getResponseBytes();
+ int entries = countZipEntries(zipContents);
+ assertEquals(5, entries); /* 2 log files + thread dump + config + stats */
+
+ // verify contents of thread dump
+ String threadContents = extractFileFromZip("threaddump.txt", zipContents);
+ assertTrue(threadContents.contains("Thread[Signal Dispatcher,"));
+ assertTrue(threadContents.contains("Thread[main,"));
+
+ // verify contents of config file
+ String configContents = extractFileFromZip("config.txt", zipContents);
+ // search for: \ngsa.version[spaces]= 7.0.14-114\n
+ int position = configContents.indexOf("\ngsa.version");
+ assertTrue(position > 0);
+ int equals = configContents.indexOf("=", position);
+ assertTrue(equals > 0);
+ assertEquals("= 7.0.14-114\n",
+ configContents.substring(equals, equals + 13));
+
+ // verify contents of stats file
+ String goldenStats = "versionJvm = 1.6.0\n\n"
+ + "isIncrementalSupported = false\n"
+ + "numTotalDocIdsPushed = 0\n\n";
+ String statsContents = extractFileFromZip("stats.txt", zipContents);
+ assertEquals(goldenStats, statsContents);
}
@Test
public void testLogFilesWithNoLogsDir() throws Exception {
- TimeZone previousTimeZone = TimeZone.getDefault();
- TimeZone.setDefault(TimeZone.getTimeZone("PST"));
- try {
- handler = new ModifiedDownloadDumpHandler(config, "myadaptor",
- new MockFile("no-such-dir").setExists(false), new MockTimeProvider());
- handler.handle(ex);
- assertEquals(200, ex.getResponseCode());
- assertEquals("attachment; filename=\"myadaptor-19691231.zip\"",
- ex.getResponseHeaders().getFirst("Content-Disposition"));
- int entries = countZipEntries(ex.getResponseBytes());
- assertEquals(2, entries); /* 0 log files + thread dump + config */
- } finally {
- TimeZone.setDefault(previousTimeZone);
- }
+ handler = new ModifiedDownloadDumpHandler(config, "myadaptor",
+ new MockFile("no-such-dir").setExists(false), new MockTimeProvider());
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("attachment; filename=\"myadaptor-19691231.zip\"",
+ ex.getResponseHeaders().getFirst("Content-Disposition"));
+ int entries = countZipEntries(ex.getResponseBytes());
+ assertEquals(2, entries); /* 0 log files + thread dump + config */
+ }
+
+ @Test
+ public void testLogFilesWithEmptyStats() throws Exception {
+ // set up File using MockFile
+ handler = new ModifiedDownloadDumpHandler(config, "myadaptor",
+ new MockFile("no-such-dir").setExists(false), new MockTimeProvider());
+ StatRpcMethod statRpcMethod = new MockStatRpcMethod(
+ new MockJournal(new MockTimeProvider()),
+ new MockAdaptor(),
+ /*isAdaptorIncremental=*/ false,
+ /*configFile=*/ new MockFile("no-such-dir").setExists(false)) {
+
+ /** generate neither versionStats nor simpleStats maps */
+ @Override
+ public Object run(List request) {
+ Map<String, Object> results = new HashMap<String, Object>();
+ Map<String, Object> otherStats = new HashMap<String, Object>();
+ otherStats.put("someBooleanValue", false);
+ otherStats.put("someLongValue", 42L);
+ // rest omitted
+ results.put("otherStats", otherStats);
+
+ return Collections.unmodifiableMap(results);
+ }
+ };
+ handler = new ModifiedDownloadDumpHandler(config, "adaptor",
+ statRpcMethod, new MockFile("no-such-dir").setExists(false),
+ new MockTimeProvider());
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("application/zip",
+ ex.getResponseHeaders().getFirst("Content-Type"));
+ // extract the zip contents and just count the number of entries
+ byte[] zipContents = ex.getResponseBytes();
+ int entries = countZipEntries(zipContents);
+ assertEquals(3, entries); /* 0 log files + thread dump + config + stats */
+
+ // verify contents of stats file
+ String goldenStats = "";
+ String statsContents = extractFileFromZip("stats.txt", zipContents);
+ assertEquals(goldenStats, statsContents);
+ }
+
+ @Test
+ public void testLogFilesWithUnusualStats() throws Exception {
+ // set up File using MockFile
+ handler = new ModifiedDownloadDumpHandler(config, "myadaptor",
+ new MockFile("no-such-dir").setExists(false), new MockTimeProvider());
+ StatRpcMethod statRpcMethod = new MockStatRpcMethod(
+ new MockJournal(new MockTimeProvider()),
+ new MockAdaptor(),
+ /*isAdaptorIncremental=*/ false,
+ /*configFile=*/ new MockFile("no-such-dir").setExists(false)) {
+
+ /** generate neither versionStats nor simpleStats maps */
+ @Override
+ public Object run(List request) {
+ Map<String, Object> results = new HashMap<String, Object>();
+ Map<String, Object> simpleStats = new TreeMap<String, Object>();
+ // keys get sorted into alphabetic order in golden map below
+ simpleStats.put("whenStarted", "non-numeric value");
+ simpleStats.put("lastSuccessfulFullPushEnd", null);
+ simpleStats.put("currentFullPushStart", 0);
+ simpleStats.put("lastSuccessfulIncremementalPushStart", -5);
+ simpleStats.put("currentIncrementalPushEnd", 1389735897000L);
+ // rest omitted
+ results.put("simpleStats", simpleStats);
+
+ return Collections.unmodifiableMap(results);
+ }
+ };
+ handler = new ModifiedDownloadDumpHandler(config, "adaptor",
+ statRpcMethod, new MockFile("no-such-dir").setExists(false),
+ new MockTimeProvider());
+ handler.handle(ex);
+ assertEquals(200, ex.getResponseCode());
+ assertEquals("application/zip",
+ ex.getResponseHeaders().getFirst("Content-Type"));
+ // extract the zip contents and just count the number of entries
+ byte[] zipContents = ex.getResponseBytes();
+ int entries = countZipEntries(zipContents);
+ assertEquals(3, entries); /* 0 log files + thread dump + config + stats */
+
+ // verify contents of stats file
+ String goldenStats = "currentFullPushStart = 0\n"
+ + "currentIncrementalPushEnd = 01/14/2014 21:44:57 +0000\n"
+ + "lastSuccessfulFullPushEnd = [null]\n"
+ + "lastSuccessfulIncremementalPushStart = -5\n"
+ + "whenStarted = non-numeric value\n\n";
+ String statsContents = extractFileFromZip("stats.txt", zipContents);
+ assertEquals(goldenStats, statsContents);
}
private MockHttpExchange createExchange(String path) {
@@ -151,6 +274,23 @@
return entries;
}
+ /**
+ * extracts the contents of a specified filename from a given zip (as a
+ * <code>String</code>)
+ */
+ private String extractFileFromZip(String filename, byte[] zipContents)
+ throws IOException {
+ ZipInputStream zis = new ZipInputStream(
+ new ByteArrayInputStream(zipContents));
+ ZipEntry ze = null;
+ while ((ze = zis.getNextEntry()) != null) {
+ if (ze.getName().equals(filename)) {
+ return IOHelper.readInputStreamToString(zis, Charset.forName("UTF-8"));
+ }
+ }
+ return null; // file not found
+ }
+
private static class ModifiedDownloadDumpHandler extends DownloadDumpHandler {
public ModifiedDownloadDumpHandler(Config config, String feedName) {
super(config, feedName, null);
@@ -161,12 +301,42 @@
super(config, feedName, null, logsDir, timeProvider);
}
+ ModifiedDownloadDumpHandler(Config config, String feedName,
+ StatRpcMethod statRpcMethod, File logsDir, TimeProvider timeProvider) {
+ super(config, feedName, statRpcMethod, logsDir, timeProvider);
+ }
+
@Override
- protected InputStream createInputStream(File file) {
+ InputStream createInputStream(File file) throws IOException {
if (!(file instanceof MockFile)) {
throw new IllegalArgumentException("implemented only for MockFile.");
}
return ((MockFile) file).createInputStream();
}
}
+
+ private static class MockStatRpcMethod extends StatRpcMethod {
+ MockStatRpcMethod(Journal journal, Adaptor adaptor,
+ boolean isAdaptorIncremental, File configFile) {
+ super(journal, adaptor, isAdaptorIncremental, configFile);
+ }
+
+ @Override
+ public Object run(List request) {
+ Map<String, Object> golden = new HashMap<String, Object>();
+ Map<String, Object> simpleStats = new HashMap<String, Object>();
+ simpleStats.put("isIncrementalSupported", false);
+ simpleStats.put("numTotalDocIdsPushed", 0L);
+ // rest omitted
+ golden.put("simpleStats", simpleStats);
+
+ Map<String, Object> versionMap = new HashMap<String, Object>();
+ versionMap.put("versionJvm", "1.6.0");
+ // rest omitted
+
+ golden.put("versionStats", versionMap);
+
+ return Collections.unmodifiableMap(golden);
+ }
+ }
}
diff --git a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java
index 0475f77..d9fbe12 100644
--- a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java
+++ b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java
@@ -59,7 +59,7 @@
Translation.STATS_VERSION_UNKNOWN.toString(locale));
versionMap.put("cwd", System.getProperty("user.dir"));
versionMap.put("configFileName",
- Translation.STATUS_NONE.toString(locale));
+ Translation.STATS_CONFIG_NONE.toString(locale));
golden.put("versionStats", versionMap);
