001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.service;
016
017 import java.util.ArrayList;
018 import java.util.Date;
019 import java.util.List;
020
021 import org.apache.hadoop.conf.Configuration;
022 import org.apache.oozie.CoordinatorJobBean;
023 import org.apache.oozie.command.coord.CoordJobMatLookupCommand;
024 import org.apache.oozie.store.CoordinatorStore;
025 import org.apache.oozie.store.StoreException;
026 import org.apache.oozie.util.XCallable;
027 import org.apache.oozie.util.XLog;
028
029 /**
030 * The coordinator Materialization Lookup trigger service schedule lookup trigger command for every interval (default is
031 * 5 minutes ). This interval could be configured through oozie configuration defined is either oozie-default.xml or
032 * oozie-site.xml using the property name oozie.service.CoordJobMatLookupTriggerService.lookup.interval
033 */
034 public class CoordJobMatLookupTriggerService implements Service {
035 public static final String CONF_PREFIX = Service.CONF_PREFIX + "CoordJobMatLookupTriggerService.";
036 /**
037 * Time interval, in seconds, at which the Job materialization service will be scheduled to run.
038 */
039 public static final String CONF_LOOKUP_INTERVAL = CONF_PREFIX + "lookup.interval";
040 /**
041 * This configuration defined the duration for which job should be materialized in future
042 */
043 public static final String CONF_MATERIALIZATION_WINDOW = CONF_PREFIX + "materialization.window";
044 /**
045 * The number of callables to be queued in a batch.
046 */
047 public static final String CONF_CALLABLE_BATCH_SIZE = CONF_PREFIX + "callable.batch.size";
048
049 private static final String INSTRUMENTATION_GROUP = "coord_job_mat_lookup";
050 private static final String INSTR_MAT_JOBS_COUNTER = "jobs";
051 private static final int CONF_LOOKUP_INTERVAL_DEFAULT = 300;
052 private static final int CONF_MATERIALIZATION_WINDOW_DEFAULT = 3600;
053
054 /**
055 * This runnable class will run in every "interval" to queue CoordJobMatLookupTriggerCommand.
056 */
057 static class CoordJobMatLookupTriggerRunnable implements Runnable {
058 private int materializationWindow;
059 private long delay = 0;
060 private List<XCallable<Void>> callables;
061 private List<XCallable<Void>> delayedCallables;
062
063 public CoordJobMatLookupTriggerRunnable(int materializationWindow) {
064 this.materializationWindow = materializationWindow;
065 }
066
067 @Override
068 public void run() {
069 runCoordJobMatLookup();
070
071 if (null != callables) {
072 boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
073 if (ret == false) {
074 XLog.getLog(getClass()).warn(
075 "Unable to queue the callables commands for CoordJobMatLookupTriggerRunnable. "
076 + "Most possibly command queue is full. Queue size is :"
077 + Services.get().get(CallableQueueService.class).queueSize());
078 }
079 callables = null;
080 }
081 if (null != delayedCallables) {
082 boolean ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
083 if (ret == false) {
084 XLog.getLog(getClass()).warn(
085 "Unable to queue the delayedCallables commands for CoordJobMatLookupTriggerRunnable. "
086 + "Most possibly Callable queue is full. Queue size is :"
087 + Services.get().get(CallableQueueService.class).queueSize());
088 }
089 delayedCallables = null;
090 this.delay = 0;
091 }
092 }
093
094 /**
095 * Recover coordinator jobs that should be materialized
096 */
097 private void runCoordJobMatLookup() {
098 XLog.Info.get().clear();
099 XLog log = XLog.getLog(getClass());
100
101 CoordinatorStore store = null;
102 try {
103 store = Services.get().get(StoreService.class).getStore(CoordinatorStore.class);
104 store.beginTrx();
105
106 // get current date
107 Date currDate = new Date(new Date().getTime() - CONF_LOOKUP_INTERVAL_DEFAULT * 100);
108 // get list of all jobs that have actions that should be
109 // materialized.
110 List<CoordinatorJobBean> materializeJobs = store.getCoordinatorJobsToBeMaterialized(currDate, 50);
111 log.debug("CoordJobMatLookupTriggerService - Curr Date= " + currDate + ", Num jobs to materialize = "
112 + materializeJobs.size());
113 for (CoordinatorJobBean coordJob : materializeJobs) {
114 Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
115 INSTR_MAT_JOBS_COUNTER, 1);
116 queueCallable(new CoordJobMatLookupCommand(coordJob.getId(), materializationWindow));
117 }
118
119 store.commitTrx();
120 }
121 catch (StoreException ex) {
122 if (store != null) {
123 store.rollbackTrx();
124 }
125 log.warn("Exception while accessing the store", ex);
126 }
127 catch (Exception ex) {
128 log.error("Exception, {0}", ex.getMessage(), ex);
129 if (store != null && store.isActive()) {
130 try {
131 store.rollbackTrx();
132 }
133 catch (RuntimeException rex) {
134 log.warn("openjpa error, {0}", rex.getMessage(), rex);
135 }
136 }
137 }
138 finally {
139 if (store != null) {
140 if (!store.isActive()) {
141 try {
142 store.closeTrx();
143 }
144 catch (RuntimeException rex) {
145 log.warn("Exception while attempting to close store", rex);
146 }
147 }
148 else {
149 log.warn("transaction is not committed or rolled back before closing entitymanager.");
150 }
151 }
152 }
153 }
154
155 /**
156 * Adds callables to a list. If the number of callables in the list reaches {@link
157 * CoordJobMatLookupTriggerService#CONF_CALLABLE_BATCH_SIZE}, the entire batch is queued and the callables list
158 * is reset.
159 *
160 * @param callable the callable to queue.
161 */
162 private void queueCallable(XCallable<Void> callable) {
163 if (callables == null) {
164 callables = new ArrayList<XCallable<Void>>();
165 }
166 callables.add(callable);
167 if (callables.size() == Services.get().getConf().getInt(CONF_CALLABLE_BATCH_SIZE, 10)) {
168 boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
169 if (ret == false) {
170 XLog.getLog(getClass()).warn(
171 "Unable to queue the callables commands for CoordJobMatLookupTriggerRunnable. "
172 + "Most possibly command queue is full. Queue size is :"
173 + Services.get().get(CallableQueueService.class).queueSize());
174 }
175 callables = new ArrayList<XCallable<Void>>();
176 }
177 }
178
179 /**
180 * Adds callables to a list. If the number of callables in the list reaches {@link
181 * CoordJobMatLookupTriggerService#CONF_CALLABLE_BATCH_SIZE}, the entire batch is queued with the delay set to
182 * the maximum delay of the callables in the list. The callables list and the delay is reset.
183 *
184 * @param callable the callable to queue.
185 * @param delay the delay for the callable.
186 */
187 private void queueCallable(XCallable<Void> callable, long delay) {
188 if (delayedCallables == null) {
189 delayedCallables = new ArrayList<XCallable<Void>>();
190 }
191 this.delay = Math.max(this.delay, delay);
192 delayedCallables.add(callable);
193 if (delayedCallables.size() == Services.get().getConf().getInt(CONF_CALLABLE_BATCH_SIZE, 10)) {
194 boolean ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
195 if (ret == false) {
196 XLog.getLog(getClass()).warn(
197 "Unable to queue the delayedCallables commands for CoordJobMatLookupTriggerRunnable. "
198 + "Most possibly Callable queue is full. Queue size is :"
199 + Services.get().get(CallableQueueService.class).queueSize());
200 }
201 delayedCallables = new ArrayList<XCallable<Void>>();
202 this.delay = 0;
203 }
204 }
205
206 }
207
208 @Override
209 public void init(Services services) throws ServiceException {
210 Configuration conf = services.getConf();
211 Runnable lookupTriggerJobsRunnable = new CoordJobMatLookupTriggerRunnable(conf.getInt(
212 CONF_MATERIALIZATION_WINDOW, CONF_MATERIALIZATION_WINDOW_DEFAULT));// Default is 1 hour
213 services.get(SchedulerService.class).schedule(lookupTriggerJobsRunnable, 10,
214 conf.getInt(CONF_LOOKUP_INTERVAL, CONF_LOOKUP_INTERVAL_DEFAULT),// Default is 5 minutes
215 SchedulerService.Unit.SEC);
216 return;
217 }
218
219 @Override
220 public void destroy() {
221 // TODO Auto-generated method stub
222
223 }
224
225 @Override
226 public Class<? extends Service> getInterface() {
227 return CoordJobMatLookupTriggerService.class;
228 }
229
230 }