From 5c25131d6e8e9ff399b6b31907d1484b486ac525 Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Fri, 1 Feb 2019 00:38:56 +0200 Subject: [PATCH 1/7] Create a function to process a single file. Signed-off-by: Tzachi Dar --- .../vfs/VFSLogFilePatternReceiver.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index 0675220e..c21e834c 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -320,12 +320,18 @@ private boolean isGZip(String fileName) { } public void run() { + ProcessSingleFile(getFileURL()); + } + + private void ProcessSingleFile(String fileUrl) { + Reader reader = null; + FileObject fileObject = null; //thread should end when we're no longer active while (reader == null && !terminated) { - int atIndex = getFileURL().indexOf("@"); - int protocolIndex = getFileURL().indexOf("://"); + int atIndex =fileUrl.indexOf("@"); + int protocolIndex = fileUrl.indexOf("://"); - String loggableFileURL = atIndex > -1 ? getFileURL().substring(0, protocolIndex + "://".length()) + "username:password" + getFileURL().substring(atIndex) : getFileURL(); + String loggableFileURL = atIndex > -1 ? fileUrl.substring(0, protocolIndex + "://".length()) + "username:password" + fileUrl.substring(atIndex) :fileUrl; getLogger().info("attempting to load file: " + loggableFileURL); try { FileSystemManager fileSystemManager = VFS.getManager(); @@ -338,7 +344,7 @@ public void run() { } synchronized (fileSystemManager) { - fileObject = fileSystemManager.resolveFile(getFileURL(), opts); + fileObject = fileSystemManager.resolveFile(fileUrl, opts); if (fileObject.exists()) { reader = new InputStreamReader(fileObject.getContent().getInputStream(), "UTF-8"); //now that we have a reader, remove additional portions of the file url (sftp passwords, etc.) @@ -397,7 +403,7 @@ public void run() { fileObject = null; } - fileObject = fileSystemManager.resolveFile(getFileURL(), opts); + fileObject = fileSystemManager.resolveFile(fileUrl, opts); } //file may not exist.. @@ -410,7 +416,7 @@ public void run() { getLogger().info(getPath() + " - unable to refresh fileobject", err); } - if (isGZip(getFileURL())) { + if (isGZip(fileUrl)) { InputStream gzipStream = new GZIPInputStream(fileObject.getContent().getInputStream()); Reader decoder = new InputStreamReader(gzipStream, "UTF-8"); BufferedReader bufferedReader = new BufferedReader(decoder); From f01ead3599b0b7b4b456a79eb2f6ae7ec1388de2 Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Sun, 3 Feb 2019 23:41:19 +0200 Subject: [PATCH 2/7] Allow to have the input as a pattern of files (for example c:\dir\file*). Chainsaw will read all the files, from the latest to the newest and then will tail the last one. Files can also be in the gzip format. Signed-off-by: Tzachi Dar --- pom.xml | 5 ++ .../vfs/VFSLogFilePatternReceiver.java | 83 +++++++++++++++++-- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index e788ab26..ed5fdf34 100644 --- a/pom.xml +++ b/pom.xml @@ -437,6 +437,11 @@ 1.16.20 provided + + commons-io + commons-io + 2.6 + diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index c21e834c..8c06fc93 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -16,6 +16,7 @@ package org.apache.log4j.chainsaw.vfs; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.vfs.*; import org.apache.commons.vfs.provider.URLFileName; import org.apache.commons.vfs.provider.sftp.SftpFileSystemConfigBuilder; @@ -26,6 +27,9 @@ import javax.swing.*; import java.awt.*; import java.io.*; +import java.util.Collections; +import java.util.Comparator; +import java.util.Vector; import java.util.zip.GZIPInputStream; /** @@ -319,11 +323,78 @@ private boolean isGZip(String fileName) { return fileName.endsWith( ".gz" ); } + // Try to sort the files based on the time that they have been created. + // We would like to read them from the latest and then continue to the newest, which we will also tail. + private void Sort(Vector filesToRead) { + if (filesToRead.size() == 0) { + return; + } + Collections.sort(filesToRead, new Comparator() { + @Override + public int compare(FileObject o1, FileObject o2) { + try { + return Long.compare(o1.getContent().getLastModifiedTime(), + o2.getContent().getLastModifiedTime()); + } catch (FileSystemException fse) { + getLogger().error("Error, could not get file time", fse); + return 0; + } + } + }); + getLogger().info("Going to read the following files"); + try { + for (FileObject f : filesToRead) { + getLogger().info( + "file = " + f.getName().getBaseName() + " " + f.getContent().getLastModifiedTime()); + } + } catch (FileSystemException fse) { + getLogger().error("Error, could not get file time", fse); + } + } + public void run() { - ProcessSingleFile(getFileURL()); + Vector filesToRead = new Vector(); + getLogger().info("starting to read " + getFileURL()); + if (!getFileURL().contains("*")) { + ProcessSingleFile(getFileURL(), null, null, false); + return; + } + String dir = FilenameUtils.getFullPathNoEndSeparator(getFileURL()); + String baseName = FilenameUtils.getName(getFileURL()); + getLogger().debug("dir = " + dir + " name " + baseName); + + try { + FileSystemManager fileSystemManager = VFS.getManager(); + FileSystemOptions opts = new FileSystemOptions(); + + try { + SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); + } catch (NoClassDefFoundError ncdfe) { + getLogger().warn("JSch not on classpath!", ncdfe); + } + + FileObject localFileObject = fileSystemManager.resolveFile(dir, opts); + FileObject[] children = localFileObject.getChildren(); + for (int i = 0; i < children.length; i++) { + if (FilenameUtils.wildcardMatchOnSystem(children[i].getName().getBaseName(), baseName)) { + filesToRead.add(children[i]); + } + } + Sort(filesToRead); + for (int i = 0; i < filesToRead.size(); i++) { + if (filesToRead.get(i).getName() instanceof URLFileName) { + URLFileName urlFileName = (URLFileName) filesToRead.get(i).getName(); + ProcessSingleFile(filesToRead.get(i).getName().toString(), urlFileName.getHostName(), + dir + baseName, i != filesToRead.size() - 1); + } + } + + } catch (FileSystemException fse) { + getLogger().info("Error processing directory " + getFileURL() + " exiting", fse); + } } - private void ProcessSingleFile(String fileUrl) { + private void ProcessSingleFile(String fileUrl, String hostName, String cannonicalPath, boolean forceNoTail) { Reader reader = null; FileObject fileObject = null; //thread should end when we're no longer active @@ -351,8 +422,8 @@ private void ProcessSingleFile(String fileUrl) { //check to see if the name is a URLFileName..if so, set file name to not include username/pass if (fileObject.getName() instanceof URLFileName) { URLFileName urlFileName = (URLFileName) fileObject.getName(); - setHost(urlFileName.getHostName()); - setPath(urlFileName.getPath()); + setHost(hostName != null ? hostName : urlFileName.getHostName() ); + setPath(cannonicalPath != null ? cannonicalPath : urlFileName.getPath()); } } else { getLogger().info(loggableFileURL + " not available - will re-attempt to load after waiting " + MISSING_FILE_RETRY_MILLIS + " millis"); @@ -461,7 +532,7 @@ private void ProcessSingleFile(String fileUrl) { if (isTailing() && fileLarger && !terminated) { getLogger().debug(getPath() + " - tailing file - file size: " + lastFileSize); } - } while (isTailing() && !terminated && !readingFinished); + } while (isTailing() && !terminated && !readingFinished && !forceNoTail); } catch (IOException ioe) { getLogger().info(getPath() + " - exception processing file", ioe); try { @@ -478,7 +549,7 @@ private void ProcessSingleFile(String fileUrl) { } catch (InterruptedException ie) { } } - } while (isAutoReconnect() && !terminated && !readingFinished); + } while (isAutoReconnect() && !terminated && !readingFinished && !forceNoTail); getLogger().debug(getPath() + " - processing complete"); } From 52457c3d6d9ca712c224164dc19a1da10c8d3fbe Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Mon, 4 Feb 2019 20:50:31 +0200 Subject: [PATCH 3/7] Move to use commons-vfs2 (Increase stability of logger). Signed-off-by: Tzachi Dar --- pom.xml | 8 ++++---- .../chainsaw/vfs/VFSLogFilePatternReceiver.java | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index ed5fdf34..eea7f92c 100644 --- a/pom.xml +++ b/pom.xml @@ -390,10 +390,10 @@ xstream 1.1.2 - - commons-vfs - commons-vfs - 1.0 + + org.apache.commons + commons-vfs2 + 2.2 commons-logging diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index 8c06fc93..ed9aa579 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -17,10 +17,10 @@ package org.apache.log4j.chainsaw.vfs; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.vfs.*; -import org.apache.commons.vfs.provider.URLFileName; -import org.apache.commons.vfs.provider.sftp.SftpFileSystemConfigBuilder; -import org.apache.commons.vfs.util.RandomAccessMode; +import org.apache.commons.vfs2.*; +import org.apache.commons.vfs2.provider.URLFileName; +import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder; +import org.apache.commons.vfs2.util.RandomAccessMode; import org.apache.log4j.chainsaw.receivers.VisualReceiver; import org.apache.log4j.varia.LogFilePatternReceiver; @@ -369,6 +369,7 @@ public void run() { try { SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); + SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false); } catch (NoClassDefFoundError ncdfe) { getLogger().warn("JSch not on classpath!", ncdfe); } @@ -410,6 +411,7 @@ private void ProcessSingleFile(String fileUrl, String hostName, String cannonica //if jsch not in classpath, can get NoClassDefFoundError here try { SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); + SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false); } catch (NoClassDefFoundError ncdfe) { getLogger().warn("JSch not on classpath!", ncdfe); } @@ -462,6 +464,7 @@ private void ProcessSingleFile(String fileUrl, String hostName, String cannonica //if jsch not in classpath, can get NoClassDefFoundError here try { SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); + SftpFileSystemConfigBuilder.getInstance( ).setUserDirIsRoot(opts, false); } catch (NoClassDefFoundError ncdfe) { getLogger().warn("JSch not on classpath!", ncdfe); } @@ -470,7 +473,6 @@ private void ProcessSingleFile(String fileUrl, String hostName, String cannonica synchronized (fileSystemManager) { if (fileObject != null) { fileObject.getFileSystem().getFileSystemManager().closeFileSystem(fileObject.getFileSystem()); - fileObject.close(); fileObject = null; } From e0f3963f0ae35ecc6007b3cf66539e46eb8940a0 Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Fri, 15 Feb 2019 01:19:40 +0200 Subject: [PATCH 4/7] Do not display the user name and password in the log and in the ui. Signed-off-by: Tzachi Dar --- .../vfs/VFSLogFilePatternReceiver.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index ed9aa579..af4aba0c 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -322,6 +322,13 @@ private class VFSReader implements Runnable { private boolean isGZip(String fileName) { return fileName.endsWith( ".gz" ); } + + private String getHostName(FileName fileName) { + if (fileName instanceof URLFileName) { + return ((URLFileName)fileName).getHostName(); + } + return "unknown"; + } // Try to sort the files based on the time that they have been created. // We would like to read them from the latest and then continue to the newest, which we will also tail. @@ -359,6 +366,12 @@ public void run() { ProcessSingleFile(getFileURL(), null, null, false); return; } + + int atIndex = getFileURL().indexOf("@"); + int protocolIndex = getFileURL().indexOf("://"); + String loggableFileURL = atIndex > -1 ? getFileURL().substring(0, protocolIndex + "://".length()) + "username:password" + getFileURL().substring(atIndex) :getFileURL(); + getLogger().info("starting to read loggableFileURL" + loggableFileURL); + String dir = FilenameUtils.getFullPathNoEndSeparator(getFileURL()); String baseName = FilenameUtils.getName(getFileURL()); getLogger().debug("dir = " + dir + " name " + baseName); @@ -375,6 +388,8 @@ public void run() { } FileObject localFileObject = fileSystemManager.resolveFile(dir, opts); + String dirName = localFileObject.getName().getPath(); + FileObject[] children = localFileObject.getChildren(); for (int i = 0; i < children.length; i++) { if (FilenameUtils.wildcardMatchOnSystem(children[i].getName().getBaseName(), baseName)) { @@ -383,15 +398,12 @@ public void run() { } Sort(filesToRead); for (int i = 0; i < filesToRead.size(); i++) { - if (filesToRead.get(i).getName() instanceof URLFileName) { - URLFileName urlFileName = (URLFileName) filesToRead.get(i).getName(); - ProcessSingleFile(filesToRead.get(i).getName().toString(), urlFileName.getHostName(), - dir + baseName, i != filesToRead.size() - 1); - } + ProcessSingleFile(filesToRead.get(i).getName().toString(), getHostName(filesToRead.get(i).getName()),//urlFileName.getHostName(), + dirName + FileName.SEPARATOR + baseName, i != filesToRead.size() - 1); } } catch (FileSystemException fse) { - getLogger().info("Error processing directory " + getFileURL() + " exiting", fse); + getLogger().info("Error processing directory " + loggableFileURL + " exiting", fse); } } @@ -422,11 +434,8 @@ private void ProcessSingleFile(String fileUrl, String hostName, String cannonica reader = new InputStreamReader(fileObject.getContent().getInputStream(), "UTF-8"); //now that we have a reader, remove additional portions of the file url (sftp passwords, etc.) //check to see if the name is a URLFileName..if so, set file name to not include username/pass - if (fileObject.getName() instanceof URLFileName) { - URLFileName urlFileName = (URLFileName) fileObject.getName(); - setHost(hostName != null ? hostName : urlFileName.getHostName() ); - setPath(cannonicalPath != null ? cannonicalPath : urlFileName.getPath()); - } + setHost(hostName != null ? hostName : getHostName(fileObject.getName()) ); + setPath(cannonicalPath != null ? cannonicalPath : fileObject.getName().getPath()); } else { getLogger().info(loggableFileURL + " not available - will re-attempt to load after waiting " + MISSING_FILE_RETRY_MILLIS + " millis"); } From b079c70ac2cadb60873dcf5b80eab78dbdbc52db Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Wed, 13 Mar 2019 01:42:16 +0200 Subject: [PATCH 5/7] Use maxLogingDays to limit the number of files that will be read. Since log files might contain a lot of data that is not relevant, the maxLogingDays allows one to limit the number of files that will be used. For example if maxLogingDays=14 files that their last modified time is older than 14 days will not be read. Signed-off-by: Tzachi Dar --- .../vfs/VFSLogFilePatternReceiver.java | 51 ++++++++++++++++++- .../VFSLogFilePatternReceiverBeanInfo.java | 2 + 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index af4aba0c..72cc51cd 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -27,8 +27,11 @@ import javax.swing.*; import java.awt.*; import java.io.*; +import java.text.SimpleDateFormat; import java.util.Collections; import java.util.Comparator; +import java.util.Date; +import java.util.ListIterator; import java.util.Vector; import java.util.zip.GZIPInputStream; @@ -149,6 +152,7 @@ public class VFSLogFilePatternReceiver extends LogFilePatternReceiver implements VisualReceiver { private boolean promptForUserInfo = false; + private String maxLogingDays = null; private Container container; private final Object waitForContainerLock = new Object(); private boolean autoReconnect; @@ -182,6 +186,37 @@ public boolean isPromptForUserInfo() { return promptForUserInfo; } + + /** + * Use maxLogingDays to limit the loging to recent messages. + * For example to see only data from 10 days, set this parameter do 10. + * + * @param maxLogingDays Don't open log files that are older than maxLogingDays + */ + public void setMaxLogingDays(String maxLogingDays) { + this.maxLogingDays = maxLogingDays; + } + + public String getMaxLogingDays() { + return maxLogingDays; + } + + private Long getStartLogingTime() { + if (getMaxLogingDays() == null) { + return 0L; + } + try { + long days = Integer.parseInt(getMaxLogingDays()); + getLogger().info("MaxloggingDays that will be used is " + days); + return System.currentTimeMillis() - days * 3600 * 24 * 1000; + } + catch (NumberFormatException e) { + getLogger().error("exception parding MaxloggingDays " + getMaxLogingDays(),e); + return 0L; + } + // Can not reach here. + } + /** * Accessor * @@ -348,11 +383,23 @@ public int compare(FileObject o1, FileObject o2) { } } }); + getLogger().info("Going to read the following files"); + SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy"); + try { - for (FileObject f : filesToRead) { + Long StartTime = getStartLogingTime(); + for (ListIterator iter = filesToRead.listIterator(); iter.hasNext(); ) { + FileObject f =iter.next(); + boolean ignoring = false; + String dates = sdf.format(new Date(f.getContent().getLastModifiedTime())); + if(f.getContent().getLastModifiedTime() < StartTime) { + iter.remove(); + ignoring = true; + } getLogger().info( - "file = " + f.getName().getBaseName() + " " + f.getContent().getLastModifiedTime()); + "file = " + f.getName().getBaseName() + " last modified time " + dates + (ignoring ? " will not be read because of MaxLogingDays" : "")); + } } catch (FileSystemException fse) { getLogger().error("Error, could not get file time", fse); diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiverBeanInfo.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiverBeanInfo.java index c3985e31..b8d5d633 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiverBeanInfo.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiverBeanInfo.java @@ -45,6 +45,8 @@ public PropertyDescriptor[] getPropertyDescriptors() { "filterExpression", VFSLogFilePatternReceiver.class), new PropertyDescriptor( "promptForUserInfo", VFSLogFilePatternReceiver.class), + new PropertyDescriptor( + "maxLogingDays", VFSLogFilePatternReceiver.class), new PropertyDescriptor("group", VFSLogFilePatternReceiver.class), }; } catch (Exception e) { From 6b63c22668420c703df99832f8185d254b7722e0 Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Wed, 13 Mar 2019 21:24:29 +0200 Subject: [PATCH 6/7] Fix a bug that caused that when reading gzip files, they have been read twice (second time was ignored, but consumed resources). Signed-off-by: Tzachi Dar --- .../org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java index 72cc51cd..9c1135aa 100644 --- a/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java +++ b/src/main/java/org/apache/log4j/chainsaw/vfs/VFSLogFilePatternReceiver.java @@ -551,6 +551,7 @@ private void ProcessSingleFile(String fileUrl, String hostName, String cannonica BufferedReader bufferedReader = new BufferedReader(decoder); process(bufferedReader); readingFinished = true; + continue; } //could have been truncated or appended to (don't do anything if same size) if (fileObject.getContent().getSize() < lastFileSize) { From 22ee868d896365cc4cf47947012e9b3bd05bb3cb Mon Sep 17 00:00:00 2001 From: Tzachi Dar Date: Wed, 13 Mar 2019 21:26:36 +0200 Subject: [PATCH 7/7] Update commons-vfs2 to it's latest version. Signed-off-by: Tzachi Dar --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eea7f92c..467199fd 100644 --- a/pom.xml +++ b/pom.xml @@ -393,7 +393,7 @@ org.apache.commons commons-vfs2 - 2.2 + 2.3 commons-logging