Add a diagnostic DownloadDumpHandler class Create a new Handler class (DownloadDumpHandler) that generates a new Zip file when it matches the URL /diagnostics-zip-dump. Currently, that file contains all the system logs as well as a thread dump, but future versions will also include a copy of the configuration as well as other diagnostics. New file src/com/google/enterprise/adaptor/DownloadDumpHandler.java contains most of the logic; the only changes to the other files are to register the new Handler (in Dashboard.java) and to provide a link that generates the Zip into resources/.../resources/index.html . In addition, new functionality for the MockFile test class (implemented by ejona@) is included. The DownloadDumpHandlerTest class uses this new functionality. Unit test suite for DownloadDumpHandler has 98% code coverage; the only line not covered is the one method overriden in the test class (to avoid real file I/O). Made only the one-argument constructor (with the feedname) public; the other is package-private (as it's only for testing the class).
diff --git a/build.xml b/build.xml index 4e4c8c9..e4f954e 100644 --- a/build.xml +++ b/build.xml
@@ -160,6 +160,7 @@ <javac srcdir="${src.dir}" destdir="${build-src.dir}" debug="true" includeantruntime="false" encoding="utf-8" target="1.6" source="1.6"> <compilerarg value="-Xlint:unchecked"/> + <compilerarg value="-Xlint:deprecation"/> <classpath refid="adaptorlib.build.classpath"/> <exclude name="${adaptor.pkg.dir}/examples/**"/> <exclude name="${adaptor.pkg.dir}/experimental/**"/>
diff --git a/resources/com/google/enterprise/adaptor/resources/index.html b/resources/com/google/enterprise/adaptor/resources/index.html index d952138..c2f3136 100755 --- a/resources/com/google/enterprise/adaptor/resources/index.html +++ b/resources/com/google/enterprise/adaptor/resources/index.html
@@ -122,6 +122,7 @@ </p> <h2>Recent Log Messages</h2> + <p><a href="../diagnostics-support.zip">Diagnostics zip file</a></p> <pre id="gaf-log"></pre> </body> </html>
diff --git a/src/com/google/enterprise/adaptor/Dashboard.java b/src/com/google/enterprise/adaptor/Dashboard.java index 3fa2e8a..3e47311 100644 --- a/src/com/google/enterprise/adaptor/Dashboard.java +++ b/src/com/google/enterprise/adaptor/Dashboard.java
@@ -86,6 +86,8 @@ secure))); addFilters(scope.createContext("/rpc", createAdminSecurityHandler( rpcHandler, config, sessionManager, secure))); + addFilters(scope.createContext("/diagnostics-support.zip", + new DownloadDumpHandler(config.getFeedName().replace('_', '-')))); addFilters(scope.createContext("/", new RedirectHandler(contextPrefix + "/dashboard"))); }
diff --git a/src/com/google/enterprise/adaptor/DownloadDumpHandler.java b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java new file mode 100644 index 0000000..9938e28 --- /dev/null +++ b/src/com/google/enterprise/adaptor/DownloadDumpHandler.java
@@ -0,0 +1,190 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.enterprise.adaptor; + +import com.google.common.annotations.VisibleForTesting; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.*; +import java.net.HttpURLConnection; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * Generates and serves a .zip file containing diagnostic information. + * + * <p>For example, it can include logs and a thread dump. + */ +class DownloadDumpHandler implements HttpHandler { + private static final Logger log + = Logger.getLogger(DownloadDumpHandler.class.getName()); + + /** To be used as part of the zip file name */ + private String feedName; + + /** Used to specify the top-level directory where logs are kept */ + private final File logsDir; + + /** Only used by the test class, to pass in a canned date */ + private final TimeProvider timeProvider; + + /** Used in handle() to generate the date portion of the zip file name; + we are implicitly using the local time zone. */ + private final DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); + + /** Default to "logs/" and System time */ + public DownloadDumpHandler(String feedName) { + this(feedName, new File("logs/"), new SystemTimeProvider()); + } + + @VisibleForTesting + DownloadDumpHandler(String feedName, File logsDir, + TimeProvider timeProvider) { + if (null == feedName) { + throw new NullPointerException(); + } + if (feedName.contains("\"")) { + throw new IllegalArgumentException( + "feedName must not contain the \" character"); + } + this.feedName = feedName; + this.logsDir = logsDir; + this.timeProvider = timeProvider; + } + + @Override + public void handle(HttpExchange ex) throws IOException { + String requestMethod = ex.getRequestMethod(); + if (!"GET".equals(requestMethod)) { + HttpExchanges.cannedRespond(ex, HttpURLConnection.HTTP_BAD_METHOD, + Translation.HTTP_BAD_METHOD); + return; + } + if (!ex.getRequestURI().getPath().equals(ex.getHttpContext().getPath())) { + HttpExchanges.cannedRespond(ex, HttpURLConnection.HTTP_NOT_FOUND, + Translation.HTTP_NOT_FOUND); + return; + } + String dateAsString; + synchronized (this) { // DateFormat.format() is not thread-safe! + dateAsString = dateFormat.format( + new Date(timeProvider.currentTimeMillis())); + } + String filename = feedName + "-" + dateAsString + ".zip"; + String contentType = "application/zip"; + ex.getResponseHeaders().set("Content-Disposition", + "attachment; filename=\"" + filename + "\""); + // stream the contents of the zip + HttpExchanges.startResponse( + ex, HttpURLConnection.HTTP_OK, contentType, true); + OutputStream os = ex.getResponseBody(); + BufferedOutputStream bos = new BufferedOutputStream(os); + ZipOutputStream zos = new ZipOutputStream(bos); + generateZipContents(logsDir, zos); + zos.close(); // NOT in a "finally" clause - connection killed on an error. + } + + private void generateZipContents(File logsDir, ZipOutputStream zos) + throws IOException { + dumpLogFiles(logsDir, zos); + dumpStackTraces(zos); + zos.flush(); + } + + private void dumpLogFiles(File logsDir, ZipOutputStream zos) + throws IOException { + File[] files = logsDir.listFiles(); + if (files == null) { + log.log(Level.FINER, "Unable to find logs directory {0}", logsDir); + return; + } + for (File f: files) { + // avoid zipping the (empty) lock file + if (f.getName().endsWith(".lck")) { + log.log(Level.FINEST, "Skipping lock file: {0}", f.getName()); + continue; + } + if (!f.isFile()) { + // TODO(myk): consider zipping up files present under subdirectories. + log.log(Level.FINEST, "Ignoring directory entry: {0}", f.getName()); + continue; + } + log.log(Level.FINEST, "Adding file: {0}/{1}", + new Object[] {logsDir, f.getName()}); + InputStream is = createInputStream(f); + try { + zos.putNextEntry(new ZipEntry(logsDir.toString() + "/" + f.getName())); + IOHelper.copyStream(is, zos); + } finally { + is.close(); + } + zos.closeEntry(); + } + } + + /** + * Output the stack trace for every running thread (sorted by thread name). + * + * <p>For example: + * <p><code> + * Thread[Reference Handler,10,system] + * java.lang.Object.wait(Native Method) + * java.lang.Object.wait(Object.java:502) + * java.lang.ref.Reference$ReferenceHandler.run(Reference.java:129) + * </code><p><code> + * Thread ... + * </code> + */ + private void dumpStackTraces(ZipOutputStream zos) throws IOException { + OutputStreamWriter writer = new OutputStreamWriter(zos, "UTF-8"); + String newline = "\n"; // so our support folks always see the same results + Map<Thread, StackTraceElement[]> allThreads = Thread.getAllStackTraces(); + Map<String, StackTraceElement[]> sortedThreads = + new TreeMap<String, StackTraceElement[]>(); + for (Map.Entry<Thread, StackTraceElement[]> me : allThreads.entrySet()) { + sortedThreads.put(me.getKey().toString(), me.getValue()); + } + zos.putNextEntry(new ZipEntry("threaddump.txt")); + for (Map.Entry<String, StackTraceElement[]> me : sortedThreads.entrySet()) { + writer.write(me.getKey()); + writer.write(newline); + for (StackTraceElement element : me.getValue()) { + writer.write(" "); + writer.write("" + element); + writer.write(newline); + } + writer.write(newline); + } + writer.flush(); + zos.closeEntry(); + } + + /** + * Method gets overriden in test class to avoid using "real" IO. + */ + @VisibleForTesting + InputStream createInputStream(File file) throws IOException { + return new FileInputStream(file); + } +}
diff --git a/src/logging.properties b/src/logging.properties index 275bded..9b5ffb6 100644 --- a/src/logging.properties +++ b/src/logging.properties
@@ -1,7 +1,12 @@ -handlers = java.util.logging.ConsoleHandler +handlers = java.util.logging.FileHandler,java.util.logging.ConsoleHandler + .level = FINER java.util.logging.ConsoleHandler.level = FINEST java.util.logging.ConsoleHandler.formatter = com.google.enterprise.adaptor.CustomFormatter # Uncomment if your terminal can't handle colors and the auto-detection # is incorrect. # com.google.enterprise.adaptor.CustomFormatter.useColor = false +java.util.logging.FileHandler.formatter=com.google.enterprise.adaptor.CustomFormatter +java.util.logging.FileHandler.pattern=logs/adaptor.%g.log +java.util.logging.FileHandler.limit=10485760 +java.util.logging.FileHandler.count=5
diff --git a/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java new file mode 100644 index 0000000..8aeac6f --- /dev/null +++ b/test/com/google/enterprise/adaptor/DownloadDumpHandlerTest.java
@@ -0,0 +1,157 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.enterprise.adaptor; + +import static org.junit.Assert.*; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.TimeZone; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Tests for {@link DownloadDumpHandler}. + */ +public class DownloadDumpHandlerTest { + private DownloadDumpHandler handler = + new ModifiedDownloadDumpHandler("adaptor"); + private String pathPrefix = "/"; + private MockHttpContext httpContext = + new MockHttpContext(handler, pathPrefix); + private MockHttpExchange ex = createExchange(""); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testNullFeedName() throws Exception { + thrown.expect(NullPointerException.class); + handler = new ModifiedDownloadDumpHandler(null); + } + + @Test + public void testIllegalFeedName() throws Exception { + thrown.expect(IllegalArgumentException.class); + handler = new ModifiedDownloadDumpHandler("bad\"name"); + } + + @Test + public void testPost() throws Exception { + ex = new MockHttpExchange("POST", pathPrefix, httpContext); + handler.handle(ex); + assertEquals(405, ex.getResponseCode()); + } + + @Test + public void testNotFound() throws Exception { + ex = createExchange("notfound"); + handler.handle(ex); + assertEquals(404, ex.getResponseCode()); + } + + @Test + public void testLogFilesWithCannedLogsDir() throws Exception { + // set up File using MockFile + TimeZone previousTimeZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("PST")); + MockFile mockLogsDir = new MockFile("parentDir").setChildren(new File[] { + new MockFile("log1.log").setFileContents("Log file 1"), + new MockFile("log1.log.lck").setFileContents("skipped lock file"), + new MockFile("log2.log").setFileContents("Log file 2"), + new MockFile("subdir").setChildren(new File[] { + new MockFile("nested.log").setFileContents("This file skipped.") + })}); + MockTimeProvider timeProvider = new MockTimeProvider(); + timeProvider.time = 1383763470000L; // November 6, 2013 @ 10:44:30 + try { + handler = new ModifiedDownloadDumpHandler("adaptor", mockLogsDir, + timeProvider); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("application/zip", + ex.getResponseHeaders().getFirst("Content-Type")); + assertEquals("attachment; filename=\"adaptor-20131106.zip\"", + ex.getResponseHeaders().getFirst("Content-Disposition")); + // extract the zip contents and just count the number of entries + int entries = countZipEntries(ex.getResponseBytes()); + assertEquals(3, entries); /* 2 expected log files + thread dump */ + } finally { + TimeZone.setDefault(previousTimeZone); + } + } + + @Test + public void testLogFilesWithNoLogsDir() throws Exception { + TimeZone previousTimeZone = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("PST")); + try { + handler = new ModifiedDownloadDumpHandler("myadaptor", + new MockFile("no-such-dir").setExists(false), new MockTimeProvider()); + handler.handle(ex); + assertEquals(200, ex.getResponseCode()); + assertEquals("attachment; filename=\"myadaptor-19691231.zip\"", + ex.getResponseHeaders().getFirst("Content-Disposition")); + int entries = countZipEntries(ex.getResponseBytes()); + assertEquals(1, entries); /* 0 expected log files + thread dump */ + } finally { + TimeZone.setDefault(previousTimeZone); + } + } + + private MockHttpExchange createExchange(String path) { + return new MockHttpExchange("GET", pathPrefix + path, httpContext); + } + + private int countZipEntries(byte[] bytes) throws IOException { + ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(bytes)); + int entries = 0; + ZipEntry nextEntry = null; + do { + nextEntry = zis.getNextEntry(); + if (null != nextEntry) { + entries++; + } + } while (nextEntry != null); + zis.close(); + return entries; + } + + private static class ModifiedDownloadDumpHandler extends DownloadDumpHandler { + + public ModifiedDownloadDumpHandler(String feedName) { + super(feedName); + } + + ModifiedDownloadDumpHandler(String feedName, File logsDir, + TimeProvider timeProvider) { + super(feedName, logsDir, timeProvider); + } + + @Override + protected InputStream createInputStream(File file) { + if (!(file instanceof MockFile)) { + throw new IllegalArgumentException("implemented only for MockFile."); + } + return ((MockFile) file).createInputStream(); + } + } +}
diff --git a/test/com/google/enterprise/adaptor/MockFile.java b/test/com/google/enterprise/adaptor/MockFile.java index 1ddf351..4c9d246 100644 --- a/test/com/google/enterprise/adaptor/MockFile.java +++ b/test/com/google/enterprise/adaptor/MockFile.java
@@ -15,26 +15,40 @@ package com.google.enterprise.adaptor; import java.io.*; +import java.nio.charset.Charset; +import java.util.Arrays; /** Mock File for testing file-related code paths. */ class MockFile extends File { + private static final Charset CHARSET = Charset.forName("UTF-8"); + private String fileContents = ""; private long lastModified; private boolean exists = true; + private boolean isFile = true; + private File[] children; public MockFile(String name) { super(name); } public Reader createReader() { - if (!exists) { + if (!exists || !isFile) { throw new IllegalStateException("File does not exist"); } return new StringReader(fileContents); } - public void setFileContents(String fileContents) { + public InputStream createInputStream() { + if (!exists || !isFile) { + throw new IllegalStateException("File does not exist"); + } + return new ByteArrayInputStream(fileContents.getBytes(CHARSET)); + } + + public MockFile setFileContents(String fileContents) { this.fileContents = fileContents; + return this; } @Override @@ -56,13 +70,33 @@ return exists; } - public void setExists(boolean exists) { + public MockFile setExists(boolean exists) { this.exists = exists; + return this; } @Override public boolean isFile() { - // This only mocks files, not directories and the like. - return exists; + return exists && isFile; + } + + @Override + public boolean isDirectory() { + return exists && !isFile; + } + + /** Marks the file and a directory, with the provided children. */ + public MockFile setChildren(File[] children) { + isFile = false; + this.children = children; + return this; + } + + @Override + public File[] listFiles() { + if (!exists || isFile) { + return null; + } + return Arrays.copyOf(children, children.length); } }