001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.util;
016
017 import java.util.HashMap;
018 import java.util.List;
019 import java.util.Map;
020 import java.util.regex.Matcher;
021 import java.util.regex.Pattern;
022
023 import java.util.Date;
024 import java.io.File;
025 import java.io.FileInputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.Writer;
029 import java.util.ArrayList;
030 import java.util.Collections;
031
032 import org.apache.oozie.util.XLog;
033 import org.apache.oozie.util.XLogReader;
034
035 /**
036 * XLogStreamer streams the given log file to logWriter after applying the given filter.
037 */
038 public class XLogStreamer {
039
040 /**
041 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks
042 * if the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by
043 * "|") jobId appName actionId token
044 */
045 public static class Filter {
046 private Map<String, Integer> logLevels;
047 private Map<String, String> filterParams;
048 private static List<String> parameters = new ArrayList<String>();
049 private boolean noFilter;
050 private Pattern filterPattern;
051
052 //TODO Patterns to be read from config file
053 private static final String DEFAULT_REGEX = "[^\\]]*";
054
055 public static final String ALLOW_ALL_REGEX = "(.*)";
056 private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
057 private static final String WHITE_SPACE_REGEX = "\\s+";
058 private static final String LOG_LEVEL_REGEX = "(\\w+)";
059 private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
060 + WHITE_SPACE_REGEX;
061 private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
062
063 public Filter() {
064 filterParams = new HashMap<String, String>();
065 for (int i = 0; i < parameters.size(); i++) {
066 filterParams.put(parameters.get(i), DEFAULT_REGEX);
067 }
068 logLevels = null;
069 noFilter = true;
070 filterPattern = null;
071 }
072
073 public void setLogLevel(String logLevel) {
074 if (logLevel != null && logLevel.trim().length() > 0) {
075 this.logLevels = new HashMap<String, Integer>();
076 String[] levels = logLevel.split("\\|");
077 for (int i = 0; i < levels.length; i++) {
078 String s = levels[i].trim().toUpperCase();
079 try {
080 XLog.Level.valueOf(s);
081 }
082 catch (Exception ex) {
083 continue;
084 }
085 this.logLevels.put(levels[i].toUpperCase(), 1);
086 }
087 }
088 }
089
090 public void setParameter(String filterParam, String value) {
091 if (filterParams.containsKey(filterParam)) {
092 noFilter = false;
093 filterParams.put(filterParam, value);
094 }
095 }
096
097 public static void defineParameter(String filterParam) {
098 parameters.add(filterParam);
099 }
100
101 public boolean isFilterPresent() {
102 if (noFilter && logLevels == null) {
103 return false;
104 }
105 return true;
106 }
107
108 /**
109 * Checks if the logLevel and logMessage goes through the logFilter.
110 *
111 * @param logParts
112 * @return
113 */
114 public boolean matches(ArrayList<String> logParts) {
115 String logLevel = logParts.get(0);
116 String logMessage = logParts.get(1);
117 if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
118 Matcher logMatcher = filterPattern.matcher(logMessage);
119 return logMatcher.matches();
120 }
121 else {
122 return false;
123 }
124 }
125
126 /**
127 * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing logLevel and
128 * logMessage if the pattern matches i.e A new log statement, else returns null.
129 *
130 * @param logLine
131 * @return Array containing log level and log message
132 */
133 public ArrayList<String> splitLogMessage(String logLine) {
134 Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
135 if (splitter.matches()) {
136 ArrayList<String> logParts = new ArrayList<String>();
137 logParts.add(splitter.group(2));// log level
138 logParts.add(splitter.group(3));// Log Message
139 return logParts;
140 }
141 else {
142 return null;
143 }
144 }
145
146 /**
147 * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be
148 * assigned if no filters are set.
149 */
150 public void constructPattern() {
151 if (noFilter && logLevels == null) {
152 filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
153 return;
154 }
155 StringBuilder sb = new StringBuilder();
156 if (noFilter) {
157 sb.append("(.*)");
158 }
159 else {
160 sb.append("(.* - ");
161 for (int i = 0; i < parameters.size(); i++) {
162 sb.append(parameters.get(i) + "\\[");
163 sb.append(filterParams.get(parameters.get(i)) + "\\] ");
164 }
165 sb.append(".*)");
166 }
167 filterPattern = Pattern.compile(sb.toString());
168 }
169
170 public static void reset() {
171 parameters.clear();
172 }
173 }
174
175 private String logFile;
176 private String logPath;
177 private Filter logFilter;
178 private Writer logWriter;
179 private long logRotation;
180
181 public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String logFile, long logRotationSecs) {
182 this.logWriter = logWriter;
183 this.logFilter = logFilter;
184 if (logFile == null) {
185 logFile = "oozie-app.log";
186 }
187 this.logFile = logFile;
188 this.logPath = logPath;
189 this.logRotation = logRotationSecs * 1000l;
190 }
191
192 /**
193 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
194 * applying the filters.
195 *
196 * @param startTime
197 * @param endTime
198 * @throws IOException
199 */
200 public void streamLog(Date startTime, Date endTime) throws IOException {
201 long startTimeMillis = 0;
202 long endTimeMillis;
203 if (startTime != null) {
204 startTimeMillis = startTime.getTime();
205 }
206 if (endTime == null) {
207 endTimeMillis = System.currentTimeMillis();
208 }
209 else {
210 endTimeMillis = endTime.getTime();
211 }
212 File dir = new File(logPath);
213 ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
214 for (int i = 0; i < fileList.size(); i++) {
215 InputStream ifs;
216 ifs = new FileInputStream(fileList.get(i).getFileName());
217 XLogReader logReader = new XLogReader(ifs, logFilter, logWriter);
218 logReader.processLog();
219 }
220 }
221
222 /**
223 * File name along with the modified time which will be used to sort later.
224 */
225 class FileInfo implements Comparable<FileInfo> {
226 String fileName;
227 long modTime;
228
229 public FileInfo(String fileName, long modTime) {
230 this.fileName = fileName;
231 this.modTime = modTime;
232 }
233
234 public String getFileName() {
235 return fileName;
236 }
237
238 public long getModTime() {
239 return modTime;
240 }
241
242 public int compareTo(FileInfo fileInfo) {
243 return (int) (this.modTime - fileInfo.modTime);
244 }
245 }
246
247 /**
248 * Gets the file list that will have the logs between startTime and endTime.
249 *
250 * @param dir
251 * @param startTime
252 * @param endTime
253 * @param logRotationTime
254 * @param logFile
255 * @return List of files to be streamed
256 */
257 private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime,
258 String logFile) {
259 String[] children = dir.list();
260 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
261 if (children == null) {
262 return fileList;
263 }
264 else {
265 for (int i = 0; i < children.length; i++) {
266 String filename = children[i];
267 if (!filename.startsWith(logFile) && !filename.equals(logFile)) {
268 continue;
269 }
270 File file = new File(dir.getAbsolutePath(), filename);
271 long modTime = file.lastModified();
272 if (modTime < startTime) {
273 continue;
274 }
275 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
276 continue;
277 }
278 fileList.add(new FileInfo(file.getAbsolutePath(), modTime));
279 }
280 }
281 Collections.sort(fileList);
282 return fileList;
283 }
284 }