001 /**
002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003 * Licensed under the Apache License, Version 2.0 (the "License");
004 * you may not use this file except in compliance with the License.
005 * You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software
010 * distributed under the License is distributed on an "AS IS" BASIS,
011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012 * See the License for the specific language governing permissions and
013 * limitations under the License. See accompanying LICENSE file.
014 */
015 package org.apache.oozie.command.coord;
016
017 import java.io.IOException;
018 import java.io.StringReader;
019 import java.util.Calendar;
020 import java.util.Date;
021 import java.util.TimeZone;
022
023 import org.apache.hadoop.conf.Configuration;
024 import org.apache.oozie.CoordinatorActionBean;
025 import org.apache.oozie.CoordinatorJobBean;
026 import org.apache.oozie.ErrorCode;
027 import org.apache.oozie.client.CoordinatorJob;
028 import org.apache.oozie.client.SLAEvent.SlaAppType;
029 import org.apache.oozie.command.CommandException;
030 import org.apache.oozie.coord.TimeUnit;
031 import org.apache.oozie.service.Service;
032 import org.apache.oozie.service.Services;
033 import org.apache.oozie.store.CoordinatorStore;
034 import org.apache.oozie.store.StoreException;
035 import org.apache.oozie.util.DateUtils;
036 import org.apache.oozie.util.Instrumentation;
037 import org.apache.oozie.util.XConfiguration;
038 import org.apache.oozie.util.XLog;
039 import org.apache.oozie.util.XmlUtils;
040 import org.apache.oozie.util.db.SLADbOperations;
041 import org.jdom.Element;
042 import org.jdom.JDOMException;
043
044 public class CoordActionMaterializeCommand extends CoordinatorCommand<Void> {
045 private String jobId;
046 private Date startTime;
047 private Date endTime;
048 private int lastActionNumber = 1; // over-ride by DB value
049 private final XLog log = XLog.getLog(getClass());
050 private String user;
051 private String group;
052 /**
053 * Default timeout for catchup jobs, in minutes, after which coordinator input check will timeout
054 */
055 public static final String CONF_DEFAULT_TIMEOUT_CATCHUP = Service.CONF_PREFIX + "coord.catchup.default.timeout";
056
057 public CoordActionMaterializeCommand(String jobId, Date startTime, Date endTime) {
058 super("coord_action_mater", "coord_action_mater", 1, XLog.STD);
059 this.jobId = jobId;
060 this.startTime = startTime;
061 this.endTime = endTime;
062 }
063
064 @Override
065 protected Void call(CoordinatorStore store) throws StoreException, CommandException {
066 // CoordinatorJobBean job = store.getCoordinatorJob(jobId, true);
067 CoordinatorJobBean job = store.getEntityManager().find(CoordinatorJobBean.class, jobId);
068 setLogInfo(job);
069 if (job.getLastActionTime() != null && job.getLastActionTime().compareTo(endTime) >= 0) {
070 log.info("ENDED Coordinator materialization for jobId = " + jobId
071 + " Action is *already* materialized for Materialization start time = " + startTime + " : Materialization end time = " + endTime + " Job status = " + job.getStatusStr());
072 return null;
073 }
074
075 if (endTime.after(job.getEndTime())) {
076 log.info("ENDED Coordinator materialization for jobId = " + jobId + " Materialization end time = " + endTime
077 + " surpasses coordinator job's end time = " + job.getEndTime() + " Job status = " + job.getStatusStr());
078 return null;
079 }
080
081 if (job.getPauseTime() != null && !startTime.before(job.getPauseTime())) {
082 log.info("ENDED Coordinator materialization for jobId = " + jobId + " Materialization start time = " + startTime
083 + " is after or equal to coordinator job's pause time = " + job.getPauseTime() + " Job status = " + job.getStatusStr());
084 // pausetime blocks real materialization - we change job's status back to RUNNING;
085 if (job.getStatus() == CoordinatorJob.Status.PREMATER) {
086 job.setStatus(CoordinatorJob.Status.RUNNING);
087 }
088 store.updateCoordinatorJob(job);
089 return null;
090 }
091
092 this.user = job.getUser();
093 this.group = job.getGroup();
094
095 if (job.getStatus().equals(CoordinatorJobBean.Status.PREMATER)) {
096 Configuration jobConf = null;
097 log.debug("start job :" + jobId + " Materialization ");
098 try {
099 jobConf = new XConfiguration(new StringReader(job.getConf()));
100 }
101 catch (IOException ioe) {
102 log.warn("Configuration parse error. read from DB :" + job.getConf(), ioe);
103 throw new CommandException(ErrorCode.E1005, ioe);
104 }
105
106 Instrumentation.Cron cron = new Instrumentation.Cron();
107 cron.start();
108 try {
109 materializeJobs(false, job, jobConf, store);
110 updateJobTable(job, store);
111 }
112 catch (CommandException ex) {
113 log.warn("Exception occurs:" + ex + " Making the job failed ");
114 job.setStatus(CoordinatorJobBean.Status.FAILED);
115 store.updateCoordinatorJob(job);
116 }
117 catch (Exception e) {
118 log.error("Excepion thrown :", e);
119 throw new CommandException(ErrorCode.E1001, e.getMessage(), e);
120 }
121 cron.stop();
122 }
123 else {
124 log.info("WARN: action is not in PREMATER state! It's in state=" + job.getStatus());
125 }
126 return null;
127 }
128
129 /**
130 * Create action instances starting from "start-time" to end-time" and store them into Action table.
131 *
132 * @param dryrun
133 * @param jobBean
134 * @param conf
135 * @param store
136 * @throws Exception
137 */
138 protected String materializeJobs(boolean dryrun, CoordinatorJobBean jobBean, Configuration conf,
139 CoordinatorStore store) throws Exception {
140 String jobXml = jobBean.getJobXml();
141 Element eJob = XmlUtils.parseXml(jobXml);
142 // TODO: always UTC?
143 TimeZone appTz = DateUtils.getTimeZone(jobBean.getTimeZone());
144 // TimeZone appTz = DateUtils.getTimeZone("UTC");
145 int frequency = jobBean.getFrequency();
146 TimeUnit freqTU = TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit"));
147 TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
148 Calendar start = Calendar.getInstance(appTz);
149 start.setTime(startTime);
150 DateUtils.moveToEnd(start, endOfFlag);
151 Calendar end = Calendar.getInstance(appTz);
152 end.setTime(endTime);
153 lastActionNumber = jobBean.getLastActionNumber();
154 // DateUtils.moveToEnd(end, endOfFlag);
155 log.info(" *** materialize Actions for tz=" + appTz.getDisplayName() + ",\n start=" + start.getTime()
156 + ", end=" + end.getTime() + "\n TimeUNIT " + freqTU.getCalendarUnit() + " Frequency :" + frequency
157 + ":" + freqTU + " lastActionNumber " + lastActionNumber);
158 // Keep the actual start time
159 Calendar origStart = Calendar.getInstance(appTz);
160 origStart.setTime(jobBean.getStartTimestamp());
161 // Move to the End of duration, if needed.
162 DateUtils.moveToEnd(origStart, endOfFlag);
163 // Cloning the start time to be used in loop iteration
164 Calendar effStart = (Calendar) origStart.clone();
165 // Move the time when the previous action finished
166 effStart.add(freqTU.getCalendarUnit(), lastActionNumber * frequency);
167
168 String action = null;
169 StringBuilder actionStrings = new StringBuilder();
170 Date jobPauseTime = jobBean.getPauseTime();
171 Calendar pause = null;
172 if (jobPauseTime != null) {
173 pause = Calendar.getInstance(appTz);
174 pause.setTime(DateUtils.convertDateToTimestamp(jobPauseTime));
175 }
176
177 while (effStart.compareTo(end) < 0) {
178 if (pause != null && effStart.compareTo(pause) >= 0) {
179 break;
180 }
181 CoordinatorActionBean actionBean = new CoordinatorActionBean();
182 lastActionNumber++;
183
184 actionBean.setTimeOut(jobBean.getTimeout());
185
186 log.debug(origStart.getTime() + " Materializing action for time=" + effStart.getTime()
187 + ", lastactionnumber=" + lastActionNumber);
188 action = CoordCommandUtils.materializeOneInstance(jobId, dryrun, (Element) eJob.clone(),
189 effStart.getTime(), lastActionNumber, conf, actionBean);
190 if (actionBean.getNominalTimestamp().before(jobBean.getCreatedTimestamp())) {
191 actionBean.setTimeOut(Services.get().getConf().getInt(CONF_DEFAULT_TIMEOUT_CATCHUP, -1));
192 log.info("Catchup timeout is :" + actionBean.getTimeOut());
193 }
194
195 if (!dryrun) {
196 storeToDB(actionBean, action, store); // Storing to table
197 }
198 else {
199 actionStrings.append("action for new instance");
200 actionStrings.append(action);
201 }
202 // Restore the original start time
203 effStart = (Calendar) origStart.clone();
204 effStart.add(freqTU.getCalendarUnit(), lastActionNumber * frequency);
205 }
206
207 endTime = new Date(effStart.getTimeInMillis());
208 if (!dryrun) {
209 return action;
210 }
211 else {
212 return actionStrings.toString();
213 }
214 }
215
216 /**
217 * Store an Action into database table.
218 *
219 * @param actionBean
220 * @param actionXml
221 * @param store
222 * @param wantSla
223 * @throws StoreException
224 * @throws JDOMException
225 */
226 private void storeToDB(CoordinatorActionBean actionBean, String actionXml, CoordinatorStore store) throws Exception {
227 log.debug("In storeToDB() action Id " + actionBean.getId() + " Size of actionXml " + actionXml.length());
228 actionBean.setActionXml(actionXml);
229 store.insertCoordinatorAction(actionBean);
230 writeActionRegistration(actionXml, actionBean, store);
231
232 // TODO: time 100s should be configurable
233 queueCallable(new CoordActionNotification(actionBean), 100);
234 queueCallable(new CoordActionInputCheckCommand(actionBean.getId()), 100);
235 }
236
237 /**
238 * @param actionXml
239 * @param actionBean
240 * @param store
241 * @throws Exception
242 */
243 private void writeActionRegistration(String actionXml, CoordinatorActionBean actionBean, CoordinatorStore store)
244 throws Exception {
245 Element eAction = XmlUtils.parseXml(actionXml);
246 Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"));
247 SLADbOperations.writeSlaRegistrationEvent(eSla, store, actionBean.getId(), SlaAppType.COORDINATOR_ACTION, user,
248 group);
249 }
250
251 /**
252 * @param job
253 * @param store
254 * @throws StoreException
255 */
256 private void updateJobTable(CoordinatorJobBean job, CoordinatorStore store) throws StoreException {
257 // TODO: why do we need this? Isn't lastMatTime enough???
258 job.setLastActionTime(endTime);
259 job.setLastActionNumber(lastActionNumber);
260 // if the job endtime == action endtime, then set status of job to
261 // succeeded
262 // we dont need to materialize this job anymore
263 Date jobEndTime = job.getEndTime();
264 if (jobEndTime.compareTo(endTime) <= 0) {
265 job.setStatus(CoordinatorJob.Status.SUCCEEDED);
266 log.info("[" + job.getId() + "]: Update status from PREMATER to SUCCEEDED");
267 }
268 else {
269 job.setStatus(CoordinatorJob.Status.RUNNING);
270 log.info("[" + job.getId() + "]: Update status from PREMATER to RUNNING");
271 }
272 job.setNextMaterializedTime(endTime);
273 store.updateCoordinatorJob(job);
274 }
275
276 @Override
277 protected Void execute(CoordinatorStore store) throws StoreException, CommandException {
278 log.info("STARTED CoordActionMaterializeCommand for jobId=" + jobId + ", startTime=" + startTime + ", endTime="
279 + endTime);
280 try {
281 if (lock(jobId)) {
282 call(store);
283 }
284 else {
285 queueCallable(new CoordActionMaterializeCommand(jobId, startTime, endTime),
286 LOCK_FAILURE_REQUEUE_INTERVAL);
287 log.warn("CoordActionMaterializeCommand lock was not acquired - failed jobId=" + jobId
288 + ". Requeing the same.");
289 }
290 }
291 catch (InterruptedException e) {
292 queueCallable(new CoordActionMaterializeCommand(jobId, startTime, endTime), LOCK_FAILURE_REQUEUE_INTERVAL);
293 log.warn("CoordActionMaterializeCommand lock acquiring failed with exception " + e.getMessage()
294 + " for jobId=" + jobId + " Requeing the same.");
295 }
296 finally {
297 log.info(" ENDED CoordActionMaterializeCommand for jobId=" + jobId + ", startTime=" + startTime
298 + ", endTime=" + endTime);
299 }
300 return null;
301 }
302
303
304
305 /**
306 * For preliminery testing. Should be removed soon
307 *
308 * @param args
309 * @throws Exception
310 */
311 public static void main(String[] args) throws Exception {
312 new Services().init();
313 try {
314 Date startTime = DateUtils.parseDateUTC("2009-02-01T01:00Z");
315 Date endTime = DateUtils.parseDateUTC("2009-02-02T01:00Z");
316 String jobId = "0000000-091207151850551-oozie-dani-C";
317 CoordActionMaterializeCommand matCmd = new CoordActionMaterializeCommand(jobId, startTime, endTime);
318 matCmd.call();
319 }
320 finally {
321 try {
322 Thread.sleep(60000);
323 }
324 catch (Exception ex) {
325 }
326 new Services().destroy();
327 }
328 }
329
330 }