Merge branch 'master' of https://code.google.com/p/plexi
diff --git a/resources/com/google/enterprise/adaptor/TranslationStrings.properties b/resources/com/google/enterprise/adaptor/TranslationStrings.properties index 38b4c82..47a9e43 100644 --- a/resources/com/google/enterprise/adaptor/TranslationStrings.properties +++ b/resources/com/google/enterprise/adaptor/TranslationStrings.properties
@@ -38,6 +38,8 @@ HTTP header. # The server encountered an error of some sort while processing the request. HTTP_INTERNAL_ERROR=Error 500: Internal Server Error +# What to display as the configuration filename when no file was used. +STATS_CONFIG_NONE=none # Shown when adaptor version is not known. STATS_VERSION_UNKNOWN=Unknown # The name of a status that monitors whether the GSA has made a request @@ -55,5 +57,3 @@ STATUS_FEED=Feed Pushing # The last data feed was interrupted before completion. STATUS_FEED_INTERRUPTED=Push was interrupted -# What to display as the configure filename when none exists. -STATUS_NONE=none
diff --git a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java index 6547b12..e97acb4 100644 --- a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java +++ b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
@@ -15,6 +15,7 @@ package com.google.enterprise.adaptor; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; @@ -24,8 +25,9 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; +import java.util.TimeZone; import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -48,7 +50,7 @@ private String feedName; /** Used to obtain statistics for one file in the .zip */ - private StatRpcMethod statRpcMethod; + private final StatRpcMethod statRpcMethod; /** Used to specify the top-level directory where logs are kept */ private final File logsDir; @@ -217,9 +219,6 @@ */ private void dumpStats(ZipOutputStream zos) throws IOException { PrintWriter writer = new PrintWriter(new OutputStreamWriter(zos, "UTF-8")); - String newline = "\n"; // so our support folks always see the same results - // LinkedHashMap maintains put() -> get() order of elements. - LinkedHashMap<String, String> stats = new LinkedHashMap<String, String>(); if (statRpcMethod == null) { return; // don't generate empty stats file } @@ -231,53 +230,43 @@ if (null != map.get("versionStats")) { @SuppressWarnings("unchecked") Map<String, Object> vMap = (Map<String, Object>) map.get("versionStats"); - // TODO(myk): determine if these strings can be shared with the CSS code - // that displays these in the dashboard - stats.put("JVM version", vMap.get("versionJvm").toString()); - stats.put("Adaptor library version", - vMap.get("versionAdaptorLibrary").toString()); - stats.put("Adaptor type", vMap.get("typeAdaptor").toString()); - stats.put("Adaptor version", vMap.get("versionAdaptor").toString()); - stats.put("Configuration file", vMap.get("configFileName").toString()); - stats.put("current directory", vMap.get("cwd").toString()); - prettyPrintMap(writer, stats); - stats.clear(); + prettyPrintMap(writer, vMap); } if (null != map.get("simpleStats")) { @SuppressWarnings("unchecked") Map<String, Object> sMap = (Map<String, Object>) map.get("simpleStats"); - stats.put("Program started at", - prettyDate(sMap.get("whenStarted"), "Unknown")); - stats.put("Last successful full push start", - prettyDate(sMap.get("lastSuccessfulFullPushStart"), "None yet")); - stats.put("Last successful full push end", - prettyDate(sMap.get("lastSuccessfulFullPushEnd"), "None yet")); - stats.put("Current full push", - prettyDate(sMap.get("currentFullPushStart"), "None in progress")); - stats.put("Last successful incremental push start", - prettyDate(sMap.get("lastSuccessfulIncrementalPushStart"), - "None yet")); - stats.put("Last successful incremental push end", - prettyDate(sMap.get("lastSuccessfulIncrementalPushEnd"), "None yet")); - stats.put("Current incremental push", - prettyDate(sMap.get("currentIncrementalPushStart"), - "None in progress")); - stats.put("Total document ids pushed", - sMap.get("numTotalDocIdsPushed").toString()); - stats.put("Unique document ids pushed", - sMap.get("numUniqueDocIdsPushed").toString()); - stats.put("GSA document requests", - sMap.get("numTotalGsaRequests").toString()); - stats.put("GSA Unique document requests", - sMap.get("numUniqueGsaRequests").toString()); - stats.put("Non-GSA document requests", - sMap.get("numTotalNonGsaRequests").toString()); - stats.put("Non-GSA Unique document requests", - sMap.get("numUniqueNonGsaRequests").toString()); - stats.put("Time resolution", sMap.get("timeResolution") + " ms"); - prettyPrintMap(writer, stats); - stats.clear(); + Set<String> expectedDateAttrs = ImmutableSet.of("whenStarted", + "lastSuccessfulFullPushStart", "lastSuccessfulFullPushEnd", + "currentFullPushStart", "lastSuccessfulIncremementalPushStart", + "lastSuccessfulIncremementalPushEnd", "currentIncrementalPushEnd"); + DateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss Z"); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + for (String key : expectedDateAttrs) { + if (!sMap.containsKey(key)) { + log.log(Level.INFO, + "Did not find expected key \"{0}\" in simpleStats", key); + continue; + } + Object value = sMap.get(key); + if (value == null) { + continue; + } + if (!(value instanceof Number)) { + log.log(Level.INFO, + "Key \"{0}\" contained non-date value \"{1}\" in simpleStats", + new Object[] {key, value}); + continue; + } + long date = ((Number) value).longValue(); + if (date <= 0) { + continue; + } + // It's a number > 0 -- assume it's a date. + // Replace value in map with a formatted date. + sMap.put(key, dateFormat.format(new Date(date))); + } + prettyPrintMap(writer, sMap); } writer.flush(); @@ -287,7 +276,7 @@ /** * Pretty-prints a map */ - void prettyPrintMap(PrintWriter writer, Map<String, String> map) { + void prettyPrintMap(PrintWriter writer, Map<String, ?> map) { int maxKeyLength = 0; for (String key : map.keySet()) { @@ -297,28 +286,13 @@ } String outputFormat = "%-" + (maxKeyLength + 1) + "s= %s%n"; - for (Map.Entry<String, String> me : map.entrySet()) { + for (Map.Entry<String, ?> me : map.entrySet()) { writer.format(outputFormat, me.getKey(), - (me.getValue() == null ? "[null]" : me.getValue())); + (me.getValue() == null ? "[null]" : me.getValue().toString())); } writer.format("%n"); } - private String prettyDate(Object date, String defaultText) { - if (null == date) { - return defaultText; - } - try { - long value = new Long(date.toString()); - if (value == 0) { - return defaultText; - } - return new Date(value).toString(); - } catch (NumberFormatException e) { - return defaultText; - } - } - /** * Method gets overriden in test class to avoid using "real" IO. */
diff --git a/src/com/google/enterprise/adaptor/StatRpcMethod.java b/src/com/google/enterprise/adaptor/StatRpcMethod.java index fe53810..ae1e519 100644 --- a/src/com/google/enterprise/adaptor/StatRpcMethod.java +++ b/src/com/google/enterprise/adaptor/StatRpcMethod.java
@@ -22,6 +22,8 @@ import java.util.Locale; import java.util.Map; import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Provides performance data when responding to @@ -33,6 +35,8 @@ private Journal journal; private boolean isAdaptorIncremental; private File configFile; + private static final Logger log + = Logger.getLogger(StatRpcMethod.class.getName()); public StatRpcMethod(Journal journal, Adaptor adaptor, boolean isAdaptorIncremental, File configFile) { @@ -154,11 +158,13 @@ canonicalConfigFile = configFile.getCanonicalFile(); } } catch (IOException e) { - // ignore error; treat file as null + log.log(Level.WARNING, "Could not determine file location for \"" + + configFile + "\"", e); + // treat file as if it were not specified -- leave it null below } return canonicalConfigFile == null ? - Translation.STATUS_NONE.toString(locale) : + Translation.STATS_CONFIG_NONE.toString(locale) : canonicalConfigFile.toString(); } }
diff --git a/src/com/google/enterprise/adaptor/Translation.java b/src/com/google/enterprise/adaptor/Translation.java index 06c0e9a..0ac47db 100644 --- a/src/com/google/enterprise/adaptor/Translation.java +++ b/src/com/google/enterprise/adaptor/Translation.java
@@ -38,6 +38,7 @@ HTTP_BAD_METHOD, HTTP_CONFLICT_INVALID_HEADER, HTTP_INTERNAL_ERROR, + STATS_CONFIG_NONE, STATS_VERSION_UNKNOWN, STATUS_CRAWLING, STATUS_CRAWLING_NO_ACCESSES_IN_PAST_DAY, @@ -45,7 +46,6 @@ STATUS_ERROR_RATE_RATE, STATUS_FEED, STATUS_FEED_INTERRUPTED, - STATUS_NONE, ; /**
diff --git a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java index be693ed..e8514ac 100644 --- a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java +++ b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
@@ -16,6 +16,7 @@ import static org.junit.Assert.*; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -25,31 +26,45 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.TimeZone; +import java.util.TreeMap; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; + /** * Tests for {@link DownloadDumpHandler}. */ public class DownloadDumpHandlerTest { - private Config config; - private DownloadDumpHandler handler; + private final Config config = new Config(); + private DownloadDumpHandler handler = + new ModifiedDownloadDumpHandler(config, "adaptor"); private final String pathPrefix = "/"; - private MockHttpContext httpContext; - private MockHttpExchange ex; + private final MockHttpContext httpContext = + new MockHttpContext(handler, pathPrefix); + private MockHttpExchange ex = createExchange(""); + private TimeZone previousTimeZone; @Rule public ExpectedException thrown = ExpectedException.none(); @Before public void setUp() { - config = new Config(); // problems arise if gsa.version is left unset config.setValue("gsa.version", "7.0.14-114"); - handler = new ModifiedDownloadDumpHandler(config, "adaptor"); - httpContext = new MockHttpContext(handler, pathPrefix); - ex = createExchange(""); + previousTimeZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("PST")); + } + + @After + public void tearDown() { + TimeZone.setDefault(previousTimeZone); } @Test @@ -87,8 +102,6 @@ @Test public void testLogFilesWithCannedLogsDir() throws Exception { // set up File using MockFile - TimeZone previousTimeZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("PST")); MockFile mockLogsDir = new MockFile("parentDir").setChildren(new File[] { new MockFile("log1.log").setFileContents("Log file 1"), new MockFile("log1.log.lck").setFileContents("skipped lock file"), @@ -98,39 +111,149 @@ })}); MockTimeProvider timeProvider = new MockTimeProvider(); timeProvider.time = 1383763470000L; // November 6, 2013 @ 10:44:30 - try { - handler = new ModifiedDownloadDumpHandler(config, "adaptor", mockLogsDir, - timeProvider); - handler.handle(ex); - assertEquals(200, ex.getResponseCode()); - assertEquals("application/zip", - ex.getResponseHeaders().getFirst("Content-Type")); - assertEquals("attachment; filename=\"adaptor-20131106.zip\"", - ex.getResponseHeaders().getFirst("Content-Disposition")); - // extract the zip contents and just count the number of entries - int entries = countZipEntries(ex.getResponseBytes()); - assertEquals(4, entries); /* 2 log files + thread dump + config */ - } finally { - TimeZone.setDefault(previousTimeZone); - } + StatRpcMethod statRpcMethod = new MockStatRpcMethod( + new MockJournal(new MockTimeProvider()), + new MockAdaptor(), + /*isAdaptorIncremental=*/ false, + /*configFile=*/ new MockFile("no-such-dir").setExists(false)); + handler = new ModifiedDownloadDumpHandler(config, "adaptor", + statRpcMethod, mockLogsDir, timeProvider); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("application/zip", + ex.getResponseHeaders().getFirst("Content-Type")); + assertEquals("attachment; filename=\"adaptor-20131106.zip\"", + ex.getResponseHeaders().getFirst("Content-Disposition")); + // extract the zip contents and just count the number of entries + byte[] zipContents = ex.getResponseBytes(); + int entries = countZipEntries(zipContents); + assertEquals(5, entries); /* 2 log files + thread dump + config + stats */ + + // verify contents of thread dump + String threadContents = extractFileFromZip("threaddump.txt", zipContents); + assertTrue(threadContents.contains("Thread[Signal Dispatcher,")); + assertTrue(threadContents.contains("Thread[main,")); + + // verify contents of config file + String configContents = extractFileFromZip("config.txt", zipContents); + // search for: \ngsa.version[spaces]= 7.0.14-114\n + int position = configContents.indexOf("\ngsa.version"); + assertTrue(position > 0); + int equals = configContents.indexOf("=", position); + assertTrue(equals > 0); + assertEquals("= 7.0.14-114\n", + configContents.substring(equals, equals + 13)); + + // verify contents of stats file + String goldenStats = "versionJvm = 1.6.0\n\n" + + "isIncrementalSupported = false\n" + + "numTotalDocIdsPushed = 0\n\n"; + String statsContents = extractFileFromZip("stats.txt", zipContents); + assertEquals(goldenStats, statsContents); } @Test public void testLogFilesWithNoLogsDir() throws Exception { - TimeZone previousTimeZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("PST")); - try { - handler = new ModifiedDownloadDumpHandler(config, "myadaptor", - new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); - handler.handle(ex); - assertEquals(200, ex.getResponseCode()); - assertEquals("attachment; filename=\"myadaptor-19691231.zip\"", - ex.getResponseHeaders().getFirst("Content-Disposition")); - int entries = countZipEntries(ex.getResponseBytes()); - assertEquals(2, entries); /* 0 log files + thread dump + config */ - } finally { - TimeZone.setDefault(previousTimeZone); - } + handler = new ModifiedDownloadDumpHandler(config, "myadaptor", + new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("attachment; filename=\"myadaptor-19691231.zip\"", + ex.getResponseHeaders().getFirst("Content-Disposition")); + int entries = countZipEntries(ex.getResponseBytes()); + assertEquals(2, entries); /* 0 log files + thread dump + config */ + } + + @Test + public void testLogFilesWithEmptyStats() throws Exception { + // set up File using MockFile + handler = new ModifiedDownloadDumpHandler(config, "myadaptor", + new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); + StatRpcMethod statRpcMethod = new MockStatRpcMethod( + new MockJournal(new MockTimeProvider()), + new MockAdaptor(), + /*isAdaptorIncremental=*/ false, + /*configFile=*/ new MockFile("no-such-dir").setExists(false)) { + + /** generate neither versionStats nor simpleStats maps */ + @Override + public Object run(List request) { + Map<String, Object> results = new HashMap<String, Object>(); + Map<String, Object> otherStats = new HashMap<String, Object>(); + otherStats.put("someBooleanValue", false); + otherStats.put("someLongValue", 42L); + // rest omitted + results.put("otherStats", otherStats); + + return Collections.unmodifiableMap(results); + } + }; + handler = new ModifiedDownloadDumpHandler(config, "adaptor", + statRpcMethod, new MockFile("no-such-dir").setExists(false), + new MockTimeProvider()); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("application/zip", + ex.getResponseHeaders().getFirst("Content-Type")); + // extract the zip contents and just count the number of entries + byte[] zipContents = ex.getResponseBytes(); + int entries = countZipEntries(zipContents); + assertEquals(3, entries); /* 0 log files + thread dump + config + stats */ + + // verify contents of stats file + String goldenStats = ""; + String statsContents = extractFileFromZip("stats.txt", zipContents); + assertEquals(goldenStats, statsContents); + } + + @Test + public void testLogFilesWithUnusualStats() throws Exception { + // set up File using MockFile + handler = new ModifiedDownloadDumpHandler(config, "myadaptor", + new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); + StatRpcMethod statRpcMethod = new MockStatRpcMethod( + new MockJournal(new MockTimeProvider()), + new MockAdaptor(), + /*isAdaptorIncremental=*/ false, + /*configFile=*/ new MockFile("no-such-dir").setExists(false)) { + + /** generate neither versionStats nor simpleStats maps */ + @Override + public Object run(List request) { + Map<String, Object> results = new HashMap<String, Object>(); + Map<String, Object> simpleStats = new TreeMap<String, Object>(); + // keys get sorted into alphabetic order in golden map below + simpleStats.put("whenStarted", "non-numeric value"); + simpleStats.put("lastSuccessfulFullPushEnd", null); + simpleStats.put("currentFullPushStart", 0); + simpleStats.put("lastSuccessfulIncremementalPushStart", -5); + simpleStats.put("currentIncrementalPushEnd", 1389735897000L); + // rest omitted + results.put("simpleStats", simpleStats); + + return Collections.unmodifiableMap(results); + } + }; + handler = new ModifiedDownloadDumpHandler(config, "adaptor", + statRpcMethod, new MockFile("no-such-dir").setExists(false), + new MockTimeProvider()); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("application/zip", + ex.getResponseHeaders().getFirst("Content-Type")); + // extract the zip contents and just count the number of entries + byte[] zipContents = ex.getResponseBytes(); + int entries = countZipEntries(zipContents); + assertEquals(3, entries); /* 0 log files + thread dump + config + stats */ + + // verify contents of stats file + String goldenStats = "currentFullPushStart = 0\n" + + "currentIncrementalPushEnd = 01/14/2014 21:44:57 +0000\n" + + "lastSuccessfulFullPushEnd = [null]\n" + + "lastSuccessfulIncremementalPushStart = -5\n" + + "whenStarted = non-numeric value\n\n"; + String statsContents = extractFileFromZip("stats.txt", zipContents); + assertEquals(goldenStats, statsContents); } private MockHttpExchange createExchange(String path) { @@ -151,6 +274,23 @@ return entries; } + /** + * extracts the contents of a specified filename from a given zip (as a + * <code>String</code>) + */ + private String extractFileFromZip(String filename, byte[] zipContents) + throws IOException { + ZipInputStream zis = new ZipInputStream( + new ByteArrayInputStream(zipContents)); + ZipEntry ze = null; + while ((ze = zis.getNextEntry()) != null) { + if (ze.getName().equals(filename)) { + return IOHelper.readInputStreamToString(zis, Charset.forName("UTF-8")); + } + } + return null; // file not found + } + private static class ModifiedDownloadDumpHandler extends DownloadDumpHandler { public ModifiedDownloadDumpHandler(Config config, String feedName) { super(config, feedName, null); @@ -161,12 +301,42 @@ super(config, feedName, null, logsDir, timeProvider); } + ModifiedDownloadDumpHandler(Config config, String feedName, + StatRpcMethod statRpcMethod, File logsDir, TimeProvider timeProvider) { + super(config, feedName, statRpcMethod, logsDir, timeProvider); + } + @Override - protected InputStream createInputStream(File file) { + InputStream createInputStream(File file) throws IOException { if (!(file instanceof MockFile)) { throw new IllegalArgumentException("implemented only for MockFile."); } return ((MockFile) file).createInputStream(); } } + + private static class MockStatRpcMethod extends StatRpcMethod { + MockStatRpcMethod(Journal journal, Adaptor adaptor, + boolean isAdaptorIncremental, File configFile) { + super(journal, adaptor, isAdaptorIncremental, configFile); + } + + @Override + public Object run(List request) { + Map<String, Object> golden = new HashMap<String, Object>(); + Map<String, Object> simpleStats = new HashMap<String, Object>(); + simpleStats.put("isIncrementalSupported", false); + simpleStats.put("numTotalDocIdsPushed", 0L); + // rest omitted + golden.put("simpleStats", simpleStats); + + Map<String, Object> versionMap = new HashMap<String, Object>(); + versionMap.put("versionJvm", "1.6.0"); + // rest omitted + + golden.put("versionStats", versionMap); + + return Collections.unmodifiableMap(golden); + } + } }
diff --git a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java index 0475f77..d9fbe12 100644 --- a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java +++ b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java
@@ -59,7 +59,7 @@ Translation.STATS_VERSION_UNKNOWN.toString(locale)); versionMap.put("cwd", System.getProperty("user.dir")); versionMap.put("configFileName", - Translation.STATUS_NONE.toString(locale)); + Translation.STATS_CONFIG_NONE.toString(locale)); golden.put("versionStats", versionMap);