001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.service;
016
017 import org.apache.hadoop.conf.Configuration;
018 import org.apache.oozie.util.Instrumentable;
019 import org.apache.oozie.util.Instrumentation;
020 import org.apache.oozie.util.XLog;
021 import org.apache.oozie.util.XConfiguration;
022 import org.apache.oozie.ErrorCode;
023
024 import java.io.File;
025 import java.io.FileInputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.StringWriter;
029 import java.util.HashSet;
030 import java.util.Map;
031 import java.util.Set;
032 import java.util.Arrays;
033
034 /**
035 * Built in service that initializes the services configuration.
036 * <p/>
037 * The configuration loading sequence is identical to Hadoop configuration loading sequence.
038 * <p/>
039 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
040 * are loaded from a site configuration file from the Oozie configuration directory.
041 * <p/>
042 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
043 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
044 * initialization of the <code>ConfigurationService</code> fails.
045 * <p/>
046 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
047 * <p/>
048 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
049 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
050 * <p/>
051 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
052 * <p/>
053 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
054 * via system properties settings.
055 */
056 public class ConfigurationService implements Service, Instrumentable {
057 private static final String INSTRUMENTATION_GROUP = "configuration";
058
059 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
060
061 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
062
063 /**
064 * System property that indicates the name of the site configuration file to load.
065 */
066 public static final String OOZIE_CONFIG_FILE_ENV = "OOZIE_CONFIG_FILE";
067
068 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
069 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
070
071 private static final String PASSWORD_PROPERTY_END = ".password";
072
073 static {
074 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE_ENV);
075 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD_ENV);
076 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
077 }
078
079 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
080 public static final String SITE_CONFIG_FILE = "oozie-site.xml";
081
082 private static XLog log = XLog.getLog(ConfigurationService.class);
083
084 private String configDir;
085 private String configFile;
086
087 private LogChangesConfiguration configuration;
088
089 public ConfigurationService() {
090 log = XLog.getLog(ConfigurationService.class);
091 }
092
093 /**
094 * Obtains the value of a system property or if not defined from an environment variable.
095 *
096 * @param envName environment variable name
097 * @param defaultValue default value if not set
098 * @return the value of the environment variable.
099 */
100 private static String getEnvValue(String envName, String defaultValue) {
101 String value = System.getProperty(envName);
102 if (value == null) {
103 value = System.getenv(envName);
104 String debugValue = (value == null) ? "variable not defined" : "value [" + value + "]";
105 log.debug("Fetching env var [{0}], {1}", envName, debugValue);
106 }
107 else {
108 log.debug("Fetching env var [{0}], Java system property overriding it, value [{1}]", envName, value);
109 }
110 return (value != null) ? value : defaultValue;
111 }
112
113 /**
114 * Initialize the log service.
115 *
116 * @param services services instance.
117 * @throws ServiceException thrown if the log service could not be initialized.
118 */
119 public void init(Services services) throws ServiceException {
120 configDir = getConfigurationDirectory();
121 configFile = getEnvValue(OOZIE_CONFIG_FILE_ENV, SITE_CONFIG_FILE);
122 if (configFile.contains("/")) {
123 throw new ServiceException(ErrorCode.E0022, configFile);
124 }
125 log.info("Oozie home [{0}]", configDir);
126 log.info("Oozie site [{0}]", configFile);
127 configFile = new File(configDir, configFile).toString();
128 configuration = loadConf();
129 }
130
131 public static String getConfigurationDirectory() throws ServiceException {
132 String oozieHome = Services.getOozieHome();
133 String configDir = new File(oozieHome, "conf").toString();
134 File file = new File(oozieHome);
135 if (!file.exists()) {
136 throw new ServiceException(ErrorCode.E0024, configDir);
137 }
138 return configDir;
139 }
140
141 /**
142 * Destroy the configuration service.
143 */
144 public void destroy() {
145 configuration = null;
146 }
147
148 /**
149 * Return the public interface for configuration service.
150 *
151 * @return {@link ConfigurationService}.
152 */
153 public Class<? extends Service> getInterface() {
154 return ConfigurationService.class;
155 }
156
157 /**
158 * Return the services configuration.
159 *
160 * @return the services configuration.
161 */
162 public Configuration getConf() {
163 if (configuration == null) {
164 throw new IllegalStateException("Not initialized");
165 }
166 return configuration;
167 }
168
169 /**
170 * Return Oozie configuration directory.
171 *
172 * @return Oozie configuration directory.
173 */
174 public String getConfDirectory() {
175 return configDir;
176 }
177
178 private InputStream getDefaultConfiguration() throws ServiceException, IOException {
179 ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
180 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
181 if (inputStream == null) {
182 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
183 }
184 return inputStream;
185 }
186
187 private LogChangesConfiguration loadConf() throws ServiceException {
188 XConfiguration configuration;
189 try {
190 InputStream inputStream = getDefaultConfiguration();
191 configuration = new XConfiguration(inputStream);
192 File file = new File(configFile);
193 if (!file.exists()) {
194 log.info("Missing site configuration file [{0}]", configFile);
195 }
196 else {
197 inputStream = new FileInputStream(configFile);
198 XConfiguration siteConfiguration = new XConfiguration(inputStream);
199 XConfiguration.injectDefaults(configuration, siteConfiguration);
200 configuration = siteConfiguration;
201 }
202 }
203 catch (IOException ex) {
204 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
205 }
206
207 if (log.isTraceEnabled()) {
208 try {
209 StringWriter writer = new StringWriter();
210 for (Map.Entry<String, String> entry : configuration) {
211 boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
212 String value = (maskValue) ? "**MASKED**" : entry.getValue();
213 writer.write(" " + entry.getKey() + " = " + value + "\n");
214 }
215 writer.close();
216 log.trace("Configuration:\n{0}---", writer.toString());
217 }
218 catch (IOException ex) {
219 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
220 }
221 }
222
223 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
224 if (ignoreSysProps != null) {
225 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
226 }
227
228 for (Map.Entry<String, String> entry : configuration) {
229 String sysValue = System.getProperty(entry.getKey());
230 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
231 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
232 configuration.set(entry.getKey(), sysValue);
233 }
234 }
235 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
236 String name = (String) entry.getKey();
237 if (IGNORE_SYS_PROPS.contains(name)) {
238 log.warn("System property [{0}] in ignore list, ignored", name);
239 }
240 else {
241 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
242 if (configuration.get(name) == null) {
243 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
244 }
245 }
246 }
247 }
248
249 return new LogChangesConfiguration(configuration);
250 }
251
252 private class LogChangesConfiguration extends XConfiguration {
253
254 public LogChangesConfiguration(Configuration conf) {
255 for (Map.Entry<String, String> entry : conf) {
256 if (get(entry.getKey()) == null) {
257 setValue(entry.getKey(), entry.getValue());
258 }
259 }
260 }
261
262 public String[] getStrings(String name) {
263 String s = get(name);
264 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
265 }
266
267 public String get(String name, String defaultValue) {
268 String value = get(name);
269 if (value == null) {
270 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
271 value = (maskValue) ? "**MASKED**" : defaultValue;
272 log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, value);
273 }
274 return value;
275 }
276
277 public void set(String name, String value) {
278 setValue(name, value);
279 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
280 value = (maskValue) ? "**MASKED**" : value;
281 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
282 }
283
284 private void setValue(String name, String value) {
285 super.set(name, value);
286 }
287
288 }
289
290 /**
291 * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
292 * config file used.
293 *
294 * @param instr instrumentation to use.
295 */
296 public void instrument(Instrumentation instr) {
297 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
298 public String getValue() {
299 return configDir;
300 }
301 });
302 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
303 public String getValue() {
304 return configFile;
305 }
306 });
307 instr.setConfiguration(configuration);
308 }
309
310 /**
311 * Return a configuration with all sensitive values masked.
312 *
313 * @param conf configuration to mask.
314 * @return masked configuration.
315 */
316 public static Configuration maskPasswords(Configuration conf) {
317 XConfiguration maskedConf = new XConfiguration();
318 for (Map.Entry<String, String> entry : conf) {
319 String name = entry.getKey();
320 boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
321 String value = (maskValue) ? "**MASKED**" : entry.getValue();
322 maskedConf.set(name, value);
323 }
324 return maskedConf;
325 }
326
327 }