|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.apache.hadoop.hive.ql.exec.Operator<T>
public abstract class Operator<T extends OperatorDesc>
Base operator implementation.
Nested Class Summary | |
---|---|
static interface |
Operator.OperatorFunc
OperatorFunc. |
static class |
Operator.ProgressCounter
TODO This is a hack for hadoop 0.17 which only supports enum counters. |
static class |
Operator.State
State. |
Field Summary | |
---|---|
protected String |
alias
|
protected long |
beginTime
|
protected List<Operator<? extends OperatorDesc>> |
childOperators
|
protected Operator<? extends OperatorDesc>[] |
childOperatorsArray
Cache childOperators in an array for faster access. |
protected int[] |
childOperatorsTag
|
protected Map<String,ExprNodeDesc> |
colExprMap
A map of output column name to input expression map. |
protected T |
conf
|
protected ArrayList<String> |
counterNames
List of counter names associated with the operator. |
protected HashMap<String,Operator.ProgressCounter> |
counterNameToEnum
Each operator has its own map of its counter names to disjoint ProgressCounter - it is populated at compile time and is read in at run-time while extracting the operator specific counts. |
protected HashMap<String,Long> |
counters
populated at runtime from hadoop counters at run time in the client. |
protected boolean |
done
|
protected static String |
fatalErrorCntr
|
protected Object |
groupKeyObject
|
protected String |
id
|
protected ObjectInspector[] |
inputObjInspectors
|
protected long |
inputRows
|
protected boolean |
isLogInfoEnabled
|
protected org.apache.commons.logging.Log |
LOG
|
protected static String |
numInputRowsCntr
|
protected static String |
numOutputRowsCntr
|
protected String |
operatorId
|
protected OutputCollector |
out
|
protected ObjectInspector |
outputObjInspector
|
protected long |
outputRows
|
protected List<Operator<? extends OperatorDesc>> |
parentOperators
|
protected Reporter |
reporter
|
protected Operator.State |
state
|
protected HashMap<Enum<?>,LongWritable> |
statsMap
|
protected static String |
timeTakenCntr
|
protected long |
totalTime
|
Constructor Summary | |
---|---|
Operator()
|
|
Operator(Reporter reporter)
Create an operator with a reporter. |
Method Summary | |
---|---|
boolean |
acceptLimitPushdown()
used for LimitPushdownOptimizer if all of the operators between limit and reduce-sink does not remove any input rows in the range of limit count, limit can be pushed down to reduce-sink operator. |
protected boolean |
allInitializedParentsAreClosed()
|
protected boolean |
areAllParentsInitialized()
checks whether all parent operators are initialized or not. |
void |
assignCounterNameToEnum()
Called only in SemanticAnalyzer after all operators have added their own set of counter names. |
void |
augmentPlan()
Called during semantic analysis as operators are being added in order to give them a chance to compute any additional plan information needed. |
boolean |
checkFatalErrors(Counters ctrs,
StringBuilder errMsg)
Recursively check this operator and its descendants to see if the fatal error counter is set to non-zero. |
void |
cleanUpInputFileChanged()
|
void |
cleanUpInputFileChangedOp()
|
Operator<? extends OperatorDesc> |
clone()
|
void |
close(boolean abort)
|
protected void |
closeOp(boolean abort)
Operator specific close routine. |
boolean |
columnNamesRowResolvedCanBeObtained()
|
String |
dump(int level)
|
String |
dump(int level,
HashSet<Integer> seenOpts)
|
void |
endGroup()
|
protected void |
fatalErrorMessage(StringBuilder errMsg,
long counterValue)
Get the fatal error message based on counter's code. |
void |
flush()
|
protected void |
forward(Object row,
ObjectInspector rowInspector)
|
protected List<String> |
getAdditionalCounters()
|
List<Operator<? extends OperatorDesc>> |
getChildOperators()
|
ArrayList<Node> |
getChildren()
Implements the getChildren function for the Node Interface. |
Map<String,ExprNodeDesc> |
getColumnExprMap()
Returns a map of output column name to input expression map Note that currently it returns only key columns for ReduceSink and GroupBy operators. |
T |
getConf()
|
Configuration |
getConfiguration()
|
ArrayList<String> |
getCounterNames()
|
HashMap<String,Operator.ProgressCounter> |
getCounterNameToEnum()
|
HashMap<String,Long> |
getCounters()
|
boolean |
getDone()
|
ExecMapperContext |
getExecContext()
|
Object |
getGroupKeyObject()
|
String |
getIdentifier()
This function is not named getId(), to make sure java serialization does NOT serialize it. |
ObjectInspector[] |
getInputObjInspectors()
|
String |
getName()
Implements the getName function for the Node Interface. |
protected long |
getNextCntr(long cntr)
|
int |
getNumChild()
|
int |
getNumParent()
|
String |
getOperatorId()
|
static String |
getOperatorName()
|
List<Operator<? extends OperatorDesc>> |
getParentOperators()
|
RowSchema |
getSchema()
|
Map<Enum<?>,Long> |
getStats()
|
abstract OperatorType |
getType()
Return the type of the specific operator among the types in OperatorType. |
String |
getWrappedCounterName(String ctrName)
|
protected void |
incrCounter(String name,
long amount)
this is called in operators in map or reduce tasks. |
protected static ObjectInspector[] |
initEvaluators(ExprNodeEvaluator[] evals,
int start,
int length,
ObjectInspector rowInspector)
Initialize an array of ExprNodeEvaluator from start, for specified length and return the result ObjectInspectors. |
protected static ObjectInspector[] |
initEvaluators(ExprNodeEvaluator[] evals,
ObjectInspector rowInspector)
Initialize an array of ExprNodeEvaluator and return the result ObjectInspectors. |
protected static StructObjectInspector |
initEvaluatorsAndReturnStruct(ExprNodeEvaluator[] evals,
List<String> outputColName,
ObjectInspector rowInspector)
Initialize an array of ExprNodeEvaluator and put the return values into a StructObjectInspector with integer field names. |
void |
initialize(Configuration hconf,
ObjectInspector[] inputOIs)
Initializes operators only if all parents have been initialized. |
protected void |
initialize(Configuration hconf,
ObjectInspector inputOI,
int parentId)
Collects all the parent's output object inspectors and calls actual initialization method. |
protected void |
initializeChildren(Configuration hconf)
Calls initialize on each of the children with outputObjetInspector as the output row format. |
void |
initializeCounters()
|
void |
initializeLocalWork(Configuration hconf)
|
protected void |
initializeOp(Configuration hconf)
Operator specific initialization. |
void |
initOperatorId()
|
boolean |
isUseBucketizedHiveInputFormat()
|
void |
jobClose(Configuration conf,
boolean success,
JobCloseFeedBack feedBack)
Unlike other operator interfaces which are called from map or reduce task, jobClose is called from the jobclient side once the job has completed. |
void |
jobCloseOp(Configuration conf,
boolean success,
JobCloseFeedBack feedBack)
|
void |
logStats()
|
boolean |
opAllowedAfterMapJoin()
|
boolean |
opAllowedBeforeMapJoin()
|
boolean |
opAllowedBeforeSortMergeJoin()
|
boolean |
opAllowedConvertMapJoin()
|
void |
passExecContext(ExecMapperContext execContext)
Pass the execContext reference to every child operator |
void |
preorderMap(Operator.OperatorFunc opFunc)
|
void |
process(Object row,
int tag)
Process the row. |
void |
processGroup(int tag)
|
abstract void |
processOp(Object row,
int tag)
Process the row. |
void |
removeChild(Operator<? extends OperatorDesc> child)
|
void |
removeChildAndAdoptItsChildren(Operator<? extends OperatorDesc> child)
Remove a child and add all of the child's children to the location of the child |
boolean |
removeChildren(int depth)
|
void |
removeParent(Operator<? extends OperatorDesc> parent)
|
void |
replaceChild(Operator<? extends OperatorDesc> child,
Operator<? extends OperatorDesc> newChild)
Replace one child with another at the same position. |
void |
replaceParent(Operator<? extends OperatorDesc> parent,
Operator<? extends OperatorDesc> newParent)
Replace one parent with another at the same position. |
void |
reset()
|
static void |
resetId()
|
static void |
resetLastEnumUsed()
|
void |
resetStats()
|
void |
setAlias(String alias)
Store the alias this operator is working on behalf of. |
void |
setChildOperators(List<Operator<? extends OperatorDesc>> childOperators)
|
void |
setColumnExprMap(Map<String,ExprNodeDesc> colExprMap)
|
void |
setConf(T conf)
|
void |
setCounterNames(ArrayList<String> counterNames)
|
void |
setCounterNameToEnum(HashMap<String,Operator.ProgressCounter> counterNameToEnum)
|
void |
setDone(boolean done)
|
void |
setExecContext(ExecMapperContext execContext)
|
void |
setGroupKeyObject(Object keyObject)
|
void |
setId(String id)
|
void |
setInputObjInspectors(ObjectInspector[] inputObjInspectors)
|
void |
setOperatorId(String operatorId)
|
void |
setOutputCollector(OutputCollector out)
|
void |
setParentOperators(List<Operator<? extends OperatorDesc>> parentOperators)
|
void |
setReporter(Reporter rep)
|
void |
setSchema(RowSchema rowSchema)
|
void |
setUseBucketizedHiveInputFormat(boolean useBucketizedHiveInputFormat)
|
void |
startGroup()
|
boolean |
supportAutomaticSortMergeJoin()
Whether this operator supports automatic sort merge join. |
boolean |
supportSkewJoinOptimization()
|
boolean |
supportUnionRemoveOptimization()
|
String |
toString()
|
static String |
toString(Collection<Operator<? extends OperatorDesc>> top)
|
void |
updateCounters(Counters ctrs)
called in ExecDriver.progress periodically. |
Methods inherited from class java.lang.Object |
---|
equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait |
Field Detail |
---|
protected List<Operator<? extends OperatorDesc>> childOperators
protected List<Operator<? extends OperatorDesc>> parentOperators
protected String operatorId
protected ArrayList<String> counterNames
protected HashMap<String,Operator.ProgressCounter> counterNameToEnum
protected transient Operator.State state
protected T extends OperatorDesc conf
protected boolean done
protected transient HashMap<Enum<?>,LongWritable> statsMap
protected transient OutputCollector out
protected transient org.apache.commons.logging.Log LOG
protected transient boolean isLogInfoEnabled
protected transient String alias
protected transient Reporter reporter
protected transient String id
protected transient ObjectInspector[] inputObjInspectors
protected transient ObjectInspector outputObjInspector
protected transient Map<String,ExprNodeDesc> colExprMap
protected transient Operator<? extends OperatorDesc>[] childOperatorsArray
protected transient int[] childOperatorsTag
protected transient HashMap<String,Long> counters
protected transient long inputRows
protected transient long outputRows
protected transient long beginTime
protected transient long totalTime
protected transient Object groupKeyObject
protected static String numInputRowsCntr
protected static String numOutputRowsCntr
protected static String timeTakenCntr
protected static String fatalErrorCntr
Constructor Detail |
---|
public Operator()
public Operator(Reporter reporter)
reporter
- Used to report progress of certain operators.Method Detail |
---|
public static void resetId()
public void setChildOperators(List<Operator<? extends OperatorDesc>> childOperators)
public Configuration getConfiguration()
public List<Operator<? extends OperatorDesc>> getChildOperators()
public int getNumChild()
public ArrayList<Node> getChildren()
getChildren
in interface Node
public void setParentOperators(List<Operator<? extends OperatorDesc>> parentOperators)
public List<Operator<? extends OperatorDesc>> getParentOperators()
public int getNumParent()
public void setConf(T conf)
public T getConf()
public boolean getDone()
public void setDone(boolean done)
public void setSchema(RowSchema rowSchema)
public RowSchema getSchema()
public void setId(String id)
public String getIdentifier()
public void setReporter(Reporter rep)
public void setOutputCollector(OutputCollector out)
public void setAlias(String alias)
public Map<Enum<?>,Long> getStats()
protected boolean areAllParentsInitialized()
public void initialize(Configuration hconf, ObjectInspector[] inputOIs) throws HiveException
hconf
- inputOIs
- input object inspector array indexes by tag id. null value is
ignored.
HiveException
public void initializeLocalWork(Configuration hconf) throws HiveException
HiveException
protected void initializeOp(Configuration hconf) throws HiveException
HiveException
protected void initializeChildren(Configuration hconf) throws HiveException
HiveException
public void passExecContext(ExecMapperContext execContext)
protected void initialize(Configuration hconf, ObjectInspector inputOI, int parentId) throws HiveException
hconf
- inputOI
- OI of the row that this parent will pass to this opparentId
- parent operator id
HiveException
public ObjectInspector[] getInputObjInspectors()
public void setInputObjInspectors(ObjectInspector[] inputObjInspectors)
public abstract void processOp(Object row, int tag) throws HiveException
row
- The object representing the row.tag
- The tag of the row usually means which parent this row comes from.
Rows with the same tag should have exactly the same rowInspector
all the time.
HiveException
public void process(Object row, int tag) throws HiveException
row
- The object representing the row.tag
- The tag of the row usually means which parent this row comes from.
Rows with the same tag should have exactly the same rowInspector
all the time.
HiveException
public void startGroup() throws HiveException
HiveException
public void endGroup() throws HiveException
HiveException
public void flush() throws HiveException
HiveException
public void processGroup(int tag) throws HiveException
HiveException
protected boolean allInitializedParentsAreClosed()
public void close(boolean abort) throws HiveException
HiveException
protected void closeOp(boolean abort) throws HiveException
HiveException
public void jobCloseOp(Configuration conf, boolean success, JobCloseFeedBack feedBack) throws HiveException
HiveException
public void jobClose(Configuration conf, boolean success, JobCloseFeedBack feedBack) throws HiveException
conf
- Configuration with with which job was submittedsuccess
- whether the job was completed successfully or not
HiveException
public void replaceChild(Operator<? extends OperatorDesc> child, Operator<? extends OperatorDesc> newChild)
child
- the old childnewChild
- the new childpublic void removeChild(Operator<? extends OperatorDesc> child)
public void removeChildAndAdoptItsChildren(Operator<? extends OperatorDesc> child) throws SemanticException
child
- If this operator is not the only parent of the child. There can be unpredictable result.
SemanticException
public void removeParent(Operator<? extends OperatorDesc> parent)
public boolean removeChildren(int depth)
public void replaceParent(Operator<? extends OperatorDesc> parent, Operator<? extends OperatorDesc> newParent)
parent
- the old parentnewParent
- the new parentprotected long getNextCntr(long cntr)
protected void forward(Object row, ObjectInspector rowInspector) throws HiveException
HiveException
public void resetStats()
public void reset()
public void preorderMap(Operator.OperatorFunc opFunc)
public void logStats()
public String getName()
getName
in interface Node
public static String getOperatorName()
public Map<String,ExprNodeDesc> getColumnExprMap()
public void setColumnExprMap(Map<String,ExprNodeDesc> colExprMap)
public String dump(int level)
public String dump(int level, HashSet<Integer> seenOpts)
protected static ObjectInspector[] initEvaluators(ExprNodeEvaluator[] evals, ObjectInspector rowInspector) throws HiveException
HiveException
protected static ObjectInspector[] initEvaluators(ExprNodeEvaluator[] evals, int start, int length, ObjectInspector rowInspector) throws HiveException
HiveException
protected static StructObjectInspector initEvaluatorsAndReturnStruct(ExprNodeEvaluator[] evals, List<String> outputColName, ObjectInspector rowInspector) throws HiveException
HiveException
protected void incrCounter(String name, long amount)
name
- amount
- public ArrayList<String> getCounterNames()
public void setCounterNames(ArrayList<String> counterNames)
public String getOperatorId()
public final String getWrappedCounterName(String ctrName)
public void initOperatorId()
public void setOperatorId(String operatorId)
public HashMap<String,Long> getCounters()
public void updateCounters(Counters ctrs)
ctrs
- counters from the running jobpublic boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg)
ctrs
- protected void fatalErrorMessage(StringBuilder errMsg, long counterValue)
errMsg
- error message should be appended to this output parameter.counterValue
- input counter code.public static void resetLastEnumUsed()
public void assignCounterNameToEnum()
public void initializeCounters()
protected List<String> getAdditionalCounters()
public HashMap<String,Operator.ProgressCounter> getCounterNameToEnum()
public void setCounterNameToEnum(HashMap<String,Operator.ProgressCounter> counterNameToEnum)
public abstract OperatorType getType()
public void setGroupKeyObject(Object keyObject)
public Object getGroupKeyObject()
public void augmentPlan()
public ExecMapperContext getExecContext()
public void setExecContext(ExecMapperContext execContext)
public void cleanUpInputFileChanged() throws HiveException
HiveException
public void cleanUpInputFileChangedOp() throws HiveException
HiveException
public boolean supportSkewJoinOptimization()
public Operator<? extends OperatorDesc> clone() throws CloneNotSupportedException
clone
in class Object
CloneNotSupportedException
public boolean columnNamesRowResolvedCanBeObtained()
public boolean isUseBucketizedHiveInputFormat()
public void setUseBucketizedHiveInputFormat(boolean useBucketizedHiveInputFormat)
public boolean supportAutomaticSortMergeJoin()
public boolean supportUnionRemoveOptimization()
public boolean opAllowedBeforeMapJoin()
public boolean opAllowedAfterMapJoin()
public boolean opAllowedConvertMapJoin()
public boolean opAllowedBeforeSortMergeJoin()
public boolean acceptLimitPushdown()
public String toString()
toString
in class Object
public static String toString(Collection<Operator<? extends OperatorDesc>> top)
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |