001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.service;
016
017 import org.apache.commons.logging.LogFactory;
018 import org.apache.log4j.LogManager;
019 import org.apache.log4j.PropertyConfigurator;
020 import org.apache.oozie.util.Instrumentable;
021 import org.apache.oozie.util.Instrumentation;
022 import org.apache.oozie.util.XLog;
023 import org.apache.oozie.util.XLogStreamer;
024 import org.apache.oozie.util.XConfiguration;
025 import org.apache.oozie.BuildInfo;
026 import org.apache.oozie.ErrorCode;
027 import org.apache.hadoop.conf.Configuration;
028
029 import java.io.File;
030 import java.io.FileInputStream;
031 import java.io.IOException;
032 import java.io.InputStream;
033 import java.io.Writer;
034 import java.net.URL;
035 import java.util.Properties;
036 import java.util.Map;
037 import java.util.Date;
038
039 /**
040 * Built-in service that initializes and manages Logging via Log4j.
041 * <p/>
042 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
043 * <p/>
044 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
045 * <p/>
046 * The Log4j configuration files must be a properties file.
047 * <p/>
048 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
049 * If the file is not found there, it is looked in the classpath.
050 * <p/>
051 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
052 * <p/>
053 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
054 * <p/>
055 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>.
056 * The default value is 10 seconds.
057 */
058 public class XLogService implements Service, Instrumentable {
059 private static final String INSTRUMENTATION_GROUP = "logging";
060
061 /**
062 * System property that indicates the log4j configuration file to load.
063 */
064 public static final String LOG4J_FILE_ENV = "OOZIE_LOG4J_FILE";
065
066 /**
067 * System property that indicates the reload interval of the configuration file.
068 */
069 public static final String LOG4J_RELOAD_ENV = "OOZIE_LOG4J_RELOAD";
070
071 /**
072 * Default value for the log4j configuration file if {@link #LOG4J_FILE_ENV} is not set.
073 */
074 public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
075
076 /**
077 * Default value for the reload interval if {@link #LOG4J_RELOAD_ENV} is not set.
078 */
079 public static final String DEFAULT_RELOAD_INTERVAL = "10";
080
081 private XLog log;
082 private long interval;
083 private boolean fromClasspath;
084 private String log4jFileName;
085 private boolean logOverWS = true;
086
087 private static final String STARTUP_MESSAGE = "{E}"
088 + " ******************************************************************************* {E}"
089 + " STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
090 + " STARTUP MSG: revision [{3}]@[{4}]{E}"
091 + "*******************************************************************************";
092
093 private String oozieLogPath;
094 private String oozieLogName;
095 private int oozieLogRotation = -1;
096
097 public XLogService() {
098 }
099
100 /**
101 * Obtains the value of a system property or if not defined from an environment variable.
102 *
103 * @param envName environment variable name
104 * @param defaultValue default value if not set
105 * @return the value of the environment variable.
106 */
107 private static String getEnvValue(String envName, String defaultValue) {
108 String value = System.getProperty(envName);
109 if (value == null) {
110 value = System.getenv(envName);
111 }
112 return (value != null) ? value : defaultValue;
113 }
114
115 /**
116 * Initialize the log service.
117 *
118 * @param services services instance.
119 * @throws ServiceException thrown if the log service could not be initialized.
120 */
121 public void init(Services services) throws ServiceException {
122 Services.getOozieHome();
123 try {
124 LogManager.resetConfiguration();
125 log4jFileName = getEnvValue(LOG4J_FILE_ENV, DEFAULT_LOG4J_PROPERTIES);
126 if (log4jFileName.contains("/")) {
127 throw new ServiceException(ErrorCode.E0011, log4jFileName);
128 }
129 if (!log4jFileName.endsWith(".properties")) {
130 throw new ServiceException(ErrorCode.E0012, log4jFileName);
131 }
132 String configPath = ConfigurationService.getConfigurationDirectory();
133 File log4jFile = new File(configPath, log4jFileName);
134 if (log4jFile.exists()) {
135 fromClasspath = false;
136 }
137 else {
138 ClassLoader cl = Thread.currentThread().getContextClassLoader();
139 URL log4jUrl = cl.getResource(log4jFileName);
140 if (log4jUrl == null) {
141 throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
142 }
143 fromClasspath = true;
144 }
145
146 if (fromClasspath) {
147 ClassLoader cl = Thread.currentThread().getContextClassLoader();
148 URL log4jUrl = cl.getResource(log4jFileName);
149 PropertyConfigurator.configure(log4jUrl);
150 }
151 else {
152 interval = Long.parseLong(getEnvValue(LOG4J_RELOAD_ENV, DEFAULT_RELOAD_INTERVAL));
153 PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
154 }
155
156 log = new XLog(LogFactory.getLog(getClass()));
157
158 log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
159 BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
160 .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
161 BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo()
162 .getProperty(BuildInfo.BUILD_VC_URL));
163
164 String from = (fromClasspath) ? "CLASSPATH" : configPath;
165 String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
166 log.info("Log4j configuration file [{0}]", log4jFileName);
167 log.info("Log4j configuration file loaded from [{0}]", from);
168 log.info("Log4j reload interval [{0}]", reload);
169
170 XLog.Info.reset();
171 XLog.Info.defineParameter(USER);
172 XLog.Info.defineParameter(GROUP);
173 XLogStreamer.Filter.reset();
174 XLogStreamer.Filter.defineParameter(USER);
175 XLogStreamer.Filter.defineParameter(GROUP);
176
177 // Getting configuration for oozie log via WS
178 ClassLoader cl = Thread.currentThread().getContextClassLoader();
179 InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
180 extractInfoForLogWebService(is);
181 }
182 catch (IOException ex) {
183 throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
184 }
185 }
186
187 private void extractInfoForLogWebService(InputStream is) throws IOException {
188 Properties props = new Properties();
189 props.load(is);
190
191 Configuration conf = new XConfiguration();
192 for (Map.Entry entry : props.entrySet()) {
193 conf.set((String) entry.getKey(), (String) entry.getValue());
194 }
195 String logFile = conf.get("log4j.appender.oozie.File");
196 if (logFile == null) {
197 log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender");
198 }
199 else {
200 logFile = logFile.trim();
201 int i = logFile.lastIndexOf("/");
202 if (i == -1) {
203 log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
204 logFile);
205 logOverWS = false;
206 }
207 else {
208 String pattern = conf.get("log4j.appender.oozie.DatePattern");
209 if (pattern == null) {
210 log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
211 logOverWS = false;
212 }
213 else {
214 pattern = pattern.trim();
215 if (pattern.endsWith("HH")) {
216 oozieLogRotation = 60 * 60;
217 }
218 else {
219 if (pattern.endsWith("dd")) {
220 oozieLogRotation = 60 * 60 * 24;
221 }
222 else {
223 log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
224 pattern);
225 logOverWS = false;
226 }
227 }
228 if (oozieLogRotation > 0) {
229 oozieLogPath = logFile.substring(0, i);
230 oozieLogName = logFile.substring(i + 1);
231 }
232 else {
233 logOverWS = false;
234 }
235 }
236 }
237 }
238 }
239
240 /**
241 * Destroy the log service.
242 */
243 public void destroy() {
244 LogManager.shutdown();
245 XLog.Info.reset();
246 XLogStreamer.Filter.reset();
247 }
248
249 /**
250 * Group log info constant.
251 */
252 public static final String USER = "USER";
253
254 /**
255 * Group log info constant.
256 */
257 public static final String GROUP = "GROUP";
258
259 /**
260 * Return the public interface for log service.
261 *
262 * @return {@link XLogService}.
263 */
264 public Class<? extends Service> getInterface() {
265 return XLogService.class;
266 }
267
268 /**
269 * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval
270 * and if loaded from the classpath.
271 *
272 * @param instr instrumentation to use.
273 */
274 public void instrument(Instrumentation instr) {
275 instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
276 public String getValue() {
277 return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
278 }
279 });
280 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
281 public String getValue() {
282 return log4jFileName;
283 }
284 });
285 instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
286 public Long getValue() {
287 return interval;
288 }
289 });
290 instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
291 public Boolean getValue() {
292 return fromClasspath;
293 }
294 });
295 instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
296 public Boolean getValue() {
297 return logOverWS;
298 }
299 });
300 }
301
302 /**
303 * Stream the log of a job.
304 *
305 * @param filter log streamer filter.
306 * @param startTime start time for log events to filter.
307 * @param endTime end time for log events to filter.
308 * @param writer writer to stream the log to.
309 * @throws IOException thrown if the log cannot be streamed.
310 */
311 public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
312 if (logOverWS) {
313 new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
314 .streamLog(startTime, endTime);
315 }
316 else {
317 writer.write("Log streaming disabled!!");
318 }
319
320 }
321
322 String getLog4jProperties() {
323 return log4jFileName;
324 }
325
326 boolean getFromClasspath() {
327 return fromClasspath;
328 }
329
330 }