Add configuration and stats to the diagnostics zip
diff --git a/resources/com/google/enterprise/adaptor/TranslationStrings.properties b/resources/com/google/enterprise/adaptor/TranslationStrings.properties index 98e1f93..38b4c82 100644 --- a/resources/com/google/enterprise/adaptor/TranslationStrings.properties +++ b/resources/com/google/enterprise/adaptor/TranslationStrings.properties
@@ -55,3 +55,5 @@ STATUS_FEED=Feed Pushing # The last data feed was interrupted before completion. STATUS_FEED_INTERRUPTED=Push was interrupted +# What to display as the configure filename when none exists. +STATUS_NONE=none
diff --git a/src/com/google/enterprise/adaptor/Config.java b/src/com/google/enterprise/adaptor/Config.java index 419b504..dabf0f7 100644 --- a/src/com/google/enterprise/adaptor/Config.java +++ b/src/com/google/enterprise/adaptor/Config.java
@@ -713,6 +713,13 @@ } /** + * Returns the current config file + */ + File getConfigFile() { + return configFile; + } + + /** * Gets all configuration values that begin with {@code prefix}, returning * them as a map with the keys having {@code prefix} removed. */
diff --git a/src/com/google/enterprise/adaptor/Dashboard.java b/src/com/google/enterprise/adaptor/Dashboard.java index 3064091..224dce5 100644 --- a/src/com/google/enterprise/adaptor/Dashboard.java +++ b/src/com/google/enterprise/adaptor/Dashboard.java
@@ -17,7 +17,6 @@ import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpServer; import java.io.*; import java.util.*; @@ -38,6 +37,7 @@ private final GsaCommunicationHandler gsaCommHandler; private final SessionManager<HttpExchange> sessionManager; private final RpcHandler rpcHandler; + private final StatRpcMethod statRpcMethod; public Dashboard(Config config, GsaCommunicationHandler gsaCommHandler, Journal journal, SessionManager<HttpExchange> sessionManager, @@ -65,8 +65,9 @@ new CheckForUpdatedConfigRpcMethod(gsaCommHandler)); rpcHandler.registerRpcMethod("encodeSensitiveValue", new EncodeSensitiveValueMethod(secureValueCodec)); - rpcHandler.registerRpcMethod("getStats", new StatRpcMethod(journal, adaptor, - gsaCommHandler.isAdaptorIncremental())); + statRpcMethod = new StatRpcMethod(journal, adaptor, + gsaCommHandler.isAdaptorIncremental(), config.getConfigFile()); + rpcHandler.registerRpcMethod("getStats", statRpcMethod); } /** Starts listening for connections to the dashboard. */ @@ -79,8 +80,8 @@ addFilters(scope.createContext("/rpc", createAdminSecurityHandler( rpcHandler, config, sessionManager, secure))); addFilters(scope.createContext("/diagnostics-support.zip", - createAdminSecurityHandler(new DownloadDumpHandler( - config.getFeedName().replace('_', '-')), + createAdminSecurityHandler(new DownloadDumpHandler(config, + config.getFeedName().replace('_', '-'), statRpcMethod), config, sessionManager, secure))); addFilters(scope.createContext("/", new RedirectHandler(dashboardContext.getPath())));
diff --git a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java index 9938e28..6547b12 100644 --- a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java +++ b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
@@ -24,6 +24,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.LinkedHashMap; import java.util.Map; import java.util.TreeMap; import java.util.logging.Level; @@ -40,9 +41,15 @@ private static final Logger log = Logger.getLogger(DownloadDumpHandler.class.getName()); + /** Used to generate the configuration output file */ + private Config config; + /** To be used as part of the zip file name */ private String feedName; + /** Used to obtain statistics for one file in the .zip */ + private StatRpcMethod statRpcMethod; + /** Used to specify the top-level directory where logs are kept */ private final File logsDir; @@ -54,13 +61,18 @@ private final DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); /** Default to "logs/" and System time */ - public DownloadDumpHandler(String feedName) { - this(feedName, new File("logs/"), new SystemTimeProvider()); + public DownloadDumpHandler(Config config, String feedName, + StatRpcMethod statRpcMethod) { + this(config, feedName, statRpcMethod, new File("logs/"), + new SystemTimeProvider()); } @VisibleForTesting - DownloadDumpHandler(String feedName, File logsDir, - TimeProvider timeProvider) { + DownloadDumpHandler(Config config, String feedName, + StatRpcMethod statRpcMethod, File logsDir, TimeProvider timeProvider) { + if (null == config) { + throw new NullPointerException(); + } if (null == feedName) { throw new NullPointerException(); } @@ -68,7 +80,9 @@ throw new IllegalArgumentException( "feedName must not contain the \" character"); } + this.config = config; this.feedName = feedName; + this.statRpcMethod = statRpcMethod; // OK to leave as null this.logsDir = logsDir; this.timeProvider = timeProvider; } @@ -109,6 +123,8 @@ throws IOException { dumpLogFiles(logsDir, zos); dumpStackTraces(zos); + dumpConfig(zos); + dumpStats(zos); zos.flush(); } @@ -181,6 +197,129 @@ } /** + * Output the configuration into the diagnostics zip + */ + private void dumpConfig(ZipOutputStream zos) throws IOException { + PrintWriter writer = new PrintWriter(new OutputStreamWriter(zos, "UTF-8")); + String newline = "\n"; // so our support folks always see the same results + TreeMap<String, String> sortedConfig = new TreeMap<String, String>(); + for (String key : config.getAllKeys()) { + sortedConfig.put(key, config.getValue(key)); + } + zos.putNextEntry(new ZipEntry("config.txt")); + prettyPrintMap(writer, sortedConfig); + writer.flush(); + zos.closeEntry(); + } + + /** + * Output the version info and statistics into the diagnostics zip + */ + private void dumpStats(ZipOutputStream zos) throws IOException { + PrintWriter writer = new PrintWriter(new OutputStreamWriter(zos, "UTF-8")); + String newline = "\n"; // so our support folks always see the same results + // LinkedHashMap maintains put() -> get() order of elements. + LinkedHashMap<String, String> stats = new LinkedHashMap<String, String>(); + if (statRpcMethod == null) { + return; // don't generate empty stats file + } + @SuppressWarnings("unchecked") + Map<String, Object> map = (Map<String, Object>) statRpcMethod.run(null); + + zos.putNextEntry(new ZipEntry("stats.txt")); + + if (null != map.get("versionStats")) { + @SuppressWarnings("unchecked") + Map<String, Object> vMap = (Map<String, Object>) map.get("versionStats"); + // TODO(myk): determine if these strings can be shared with the CSS code + // that displays these in the dashboard + stats.put("JVM version", vMap.get("versionJvm").toString()); + stats.put("Adaptor library version", + vMap.get("versionAdaptorLibrary").toString()); + stats.put("Adaptor type", vMap.get("typeAdaptor").toString()); + stats.put("Adaptor version", vMap.get("versionAdaptor").toString()); + stats.put("Configuration file", vMap.get("configFileName").toString()); + stats.put("current directory", vMap.get("cwd").toString()); + prettyPrintMap(writer, stats); + stats.clear(); + } + + if (null != map.get("simpleStats")) { + @SuppressWarnings("unchecked") + Map<String, Object> sMap = (Map<String, Object>) map.get("simpleStats"); + stats.put("Program started at", + prettyDate(sMap.get("whenStarted"), "Unknown")); + stats.put("Last successful full push start", + prettyDate(sMap.get("lastSuccessfulFullPushStart"), "None yet")); + stats.put("Last successful full push end", + prettyDate(sMap.get("lastSuccessfulFullPushEnd"), "None yet")); + stats.put("Current full push", + prettyDate(sMap.get("currentFullPushStart"), "None in progress")); + stats.put("Last successful incremental push start", + prettyDate(sMap.get("lastSuccessfulIncrementalPushStart"), + "None yet")); + stats.put("Last successful incremental push end", + prettyDate(sMap.get("lastSuccessfulIncrementalPushEnd"), "None yet")); + stats.put("Current incremental push", + prettyDate(sMap.get("currentIncrementalPushStart"), + "None in progress")); + stats.put("Total document ids pushed", + sMap.get("numTotalDocIdsPushed").toString()); + stats.put("Unique document ids pushed", + sMap.get("numUniqueDocIdsPushed").toString()); + stats.put("GSA document requests", + sMap.get("numTotalGsaRequests").toString()); + stats.put("GSA Unique document requests", + sMap.get("numUniqueGsaRequests").toString()); + stats.put("Non-GSA document requests", + sMap.get("numTotalNonGsaRequests").toString()); + stats.put("Non-GSA Unique document requests", + sMap.get("numUniqueNonGsaRequests").toString()); + stats.put("Time resolution", sMap.get("timeResolution") + " ms"); + prettyPrintMap(writer, stats); + stats.clear(); + } + + writer.flush(); + zos.closeEntry(); + } + + /** + * Pretty-prints a map + */ + void prettyPrintMap(PrintWriter writer, Map<String, String> map) { + int maxKeyLength = 0; + + for (String key : map.keySet()) { + if (key.length() > maxKeyLength) { + maxKeyLength = key.length(); + } + } + + String outputFormat = "%-" + (maxKeyLength + 1) + "s= %s%n"; + for (Map.Entry<String, String> me : map.entrySet()) { + writer.format(outputFormat, me.getKey(), + (me.getValue() == null ? "[null]" : me.getValue())); + } + writer.format("%n"); + } + + private String prettyDate(Object date, String defaultText) { + if (null == date) { + return defaultText; + } + try { + long value = new Long(date.toString()); + if (value == 0) { + return defaultText; + } + return new Date(value).toString(); + } catch (NumberFormatException e) { + return defaultText; + } + } + + /** * Method gets overriden in test class to avoid using "real" IO. */ @VisibleForTesting
diff --git a/src/com/google/enterprise/adaptor/StatRpcMethod.java b/src/com/google/enterprise/adaptor/StatRpcMethod.java index 93f1a9f..fe53810 100644 --- a/src/com/google/enterprise/adaptor/StatRpcMethod.java +++ b/src/com/google/enterprise/adaptor/StatRpcMethod.java
@@ -14,6 +14,9 @@ package com.google.enterprise.adaptor; +import java.io.File; +import java.io.IOException; + import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -29,11 +32,13 @@ private String adaptorType; private Journal journal; private boolean isAdaptorIncremental; + private File configFile; public StatRpcMethod(Journal journal, Adaptor adaptor, - boolean isAdaptorIncremental) { + boolean isAdaptorIncremental, File configFile) { this.journal = journal; this.isAdaptorIncremental = isAdaptorIncremental; + this.configFile = configFile; Class adaptorClass = adaptor.getClass(); if (adaptorClass.getPackage() != null) { @@ -84,7 +89,8 @@ versionMap.put("versionAdaptorLibrary", getAdaptorLibraryVersion(locale)); versionMap.put("typeAdaptor", adaptorType); versionMap.put("versionAdaptor", getAdaptorVersion(locale)); - + versionMap.put("cwd", System.getProperty("user.dir")); + versionMap.put("configFileName", getConfigFilename(locale)); map.put("versionStats", versionMap); } @@ -140,4 +146,19 @@ return adaptorVersion == null ? Translation.STATS_VERSION_UNKNOWN.toString(locale) : adaptorVersion; } + + private String getConfigFilename(Locale locale) { + File canonicalConfigFile = null; + try { + if (configFile != null) { + canonicalConfigFile = configFile.getCanonicalFile(); + } + } catch (IOException e) { + // ignore error; treat file as null + } + + return canonicalConfigFile == null ? + Translation.STATUS_NONE.toString(locale) : + canonicalConfigFile.toString(); + } }
diff --git a/src/com/google/enterprise/adaptor/Translation.java b/src/com/google/enterprise/adaptor/Translation.java index 9f7eb37..06c0e9a 100644 --- a/src/com/google/enterprise/adaptor/Translation.java +++ b/src/com/google/enterprise/adaptor/Translation.java
@@ -45,6 +45,7 @@ STATUS_ERROR_RATE_RATE, STATUS_FEED, STATUS_FEED_INTERRUPTED, + STATUS_NONE, ; /**
diff --git a/test/com/google/enterprise/adaptor/ConfigTest.java b/test/com/google/enterprise/adaptor/ConfigTest.java index 40a66ec..d4bfe5f 100644 --- a/test/com/google/enterprise/adaptor/ConfigTest.java +++ b/test/com/google/enterprise/adaptor/ConfigTest.java
@@ -72,6 +72,7 @@ config.load(configFile); assertEquals("notreal", config.getGsaHostname()); assertEquals("1", config.getAdaptorFullListingSchedule()); + assertEquals(configFile, config.getConfigFile()); final List<ConfigModificationEvent> events = new LinkedList<ConfigModificationEvent>();
diff --git a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java index 8aeac6f..be693ed 100644 --- a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java +++ b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
@@ -16,6 +16,7 @@ import static org.junit.Assert.*; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -32,26 +33,41 @@ * Tests for {@link DownloadDumpHandler}. */ public class DownloadDumpHandlerTest { - private DownloadDumpHandler handler = - new ModifiedDownloadDumpHandler("adaptor"); - private String pathPrefix = "/"; - private MockHttpContext httpContext = - new MockHttpContext(handler, pathPrefix); - private MockHttpExchange ex = createExchange(""); + private Config config; + private DownloadDumpHandler handler; + private final String pathPrefix = "/"; + private MockHttpContext httpContext; + private MockHttpExchange ex; @Rule public ExpectedException thrown = ExpectedException.none(); + @Before + public void setUp() { + config = new Config(); + // problems arise if gsa.version is left unset + config.setValue("gsa.version", "7.0.14-114"); + handler = new ModifiedDownloadDumpHandler(config, "adaptor"); + httpContext = new MockHttpContext(handler, pathPrefix); + ex = createExchange(""); + } + + @Test + public void testNullConfig() throws Exception { + thrown.expect(NullPointerException.class); + handler = new DownloadDumpHandler(null, "feedname", null); + } + @Test public void testNullFeedName() throws Exception { thrown.expect(NullPointerException.class); - handler = new ModifiedDownloadDumpHandler(null); + handler = new ModifiedDownloadDumpHandler(config, null); } @Test public void testIllegalFeedName() throws Exception { thrown.expect(IllegalArgumentException.class); - handler = new ModifiedDownloadDumpHandler("bad\"name"); + handler = new ModifiedDownloadDumpHandler(config, "bad\"name"); } @Test @@ -83,7 +99,7 @@ MockTimeProvider timeProvider = new MockTimeProvider(); timeProvider.time = 1383763470000L; // November 6, 2013 @ 10:44:30 try { - handler = new ModifiedDownloadDumpHandler("adaptor", mockLogsDir, + handler = new ModifiedDownloadDumpHandler(config, "adaptor", mockLogsDir, timeProvider); handler.handle(ex); assertEquals(200, ex.getResponseCode()); @@ -93,7 +109,7 @@ ex.getResponseHeaders().getFirst("Content-Disposition")); // extract the zip contents and just count the number of entries int entries = countZipEntries(ex.getResponseBytes()); - assertEquals(3, entries); /* 2 expected log files + thread dump */ + assertEquals(4, entries); /* 2 log files + thread dump + config */ } finally { TimeZone.setDefault(previousTimeZone); } @@ -104,14 +120,14 @@ TimeZone previousTimeZone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("PST")); try { - handler = new ModifiedDownloadDumpHandler("myadaptor", + handler = new ModifiedDownloadDumpHandler(config, "myadaptor", new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); handler.handle(ex); assertEquals(200, ex.getResponseCode()); assertEquals("attachment; filename=\"myadaptor-19691231.zip\"", ex.getResponseHeaders().getFirst("Content-Disposition")); int entries = countZipEntries(ex.getResponseBytes()); - assertEquals(1, entries); /* 0 expected log files + thread dump */ + assertEquals(2, entries); /* 0 log files + thread dump + config */ } finally { TimeZone.setDefault(previousTimeZone); } @@ -136,14 +152,13 @@ } private static class ModifiedDownloadDumpHandler extends DownloadDumpHandler { - - public ModifiedDownloadDumpHandler(String feedName) { - super(feedName); + public ModifiedDownloadDumpHandler(Config config, String feedName) { + super(config, feedName, null); } - ModifiedDownloadDumpHandler(String feedName, File logsDir, + ModifiedDownloadDumpHandler(Config config, String feedName, File logsDir, TimeProvider timeProvider) { - super(feedName, logsDir, timeProvider); + super(config, feedName, null, logsDir, timeProvider); } @Override
diff --git a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java index fffd9f7..0475f77 100644 --- a/test/com/google/enterprise/adaptor/StatRpcMethodTest.java +++ b/test/com/google/enterprise/adaptor/StatRpcMethodTest.java
@@ -25,7 +25,7 @@ */ public class StatRpcMethodTest { private RpcHandler.RpcMethod method = new StatRpcMethod( - new SnapshotMockJournal(), new AdaptorMock(), false); + new SnapshotMockJournal(), new AdaptorMock(), false, null); @Test public void testStat() throws Exception { @@ -57,6 +57,9 @@ versionMap.put("typeAdaptor", AdaptorMock.class.getSimpleName()); versionMap.put("versionAdaptor", Translation.STATS_VERSION_UNKNOWN.toString(locale)); + versionMap.put("cwd", System.getProperty("user.dir")); + versionMap.put("configFileName", + Translation.STATUS_NONE.toString(locale)); golden.put("versionStats", versionMap);