com
.
qubole
.
spark
.
hiveacid
.
streaming
HiveAcidSinkLog
Related Docs:
object HiveAcidSinkLog
|
package streaming
class
HiveAcidSinkLog
extends
CompactibleFileStreamLog
[
HiveAcidSinkStatus
]
Linear Supertypes
CompactibleFileStreamLog
[
HiveAcidSinkStatus
],
HDFSMetadataLog
[
Array
[
HiveAcidSinkStatus
]],
Logging
,
MetadataLog
[
Array
[
HiveAcidSinkStatus
]],
AnyRef
,
Any
Ordering
Alphabetic
By Inheritance
Inherited
HiveAcidSinkLog
CompactibleFileStreamLog
HDFSMetadataLog
Logging
MetadataLog
AnyRef
Any
Hide All
Show All
Visibility
Public
All
Instance Constructors
new
HiveAcidSinkLog
(
version:
Int
,
sparkSession:
SparkSession
,
path:
String
,
options:
HiveAcidSinkOptions
)
Value Members
final
def
!=
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
final
def
##
()
:
Int
Definition Classes
AnyRef → Any
final
def
==
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
def
add
(
batchId:
Long
,
logs:
Array
[
HiveAcidSinkStatus
]
)
:
Boolean
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog → MetadataLog
def
addNewBatchByStream
(
batchId:
Long
)
(
fn: (
OutputStream
) ⇒
Unit
)
:
Boolean
Definition Classes
HDFSMetadataLog
def
allFiles
()
:
Array
[
HiveAcidSinkStatus
]
Definition Classes
CompactibleFileStreamLog
def
applyFnToBatchByStream
[
RET
]
(
batchId:
Long
)
(
fn: (
InputStream
) ⇒
RET
)
:
RET
Definition Classes
HDFSMetadataLog
final
def
asInstanceOf
[
T0
]
:
T0
Definition Classes
Any
val
batchFilesFilter
:
PathFilter
Attributes
protected
Definition Classes
HDFSMetadataLog
def
batchIdToPath
(
batchId:
Long
)
:
Path
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog
def
clone
()
:
AnyRef
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
lazy val
compactInterval
:
Int
Attributes
protected
Definition Classes
CompactibleFileStreamLog
val
defaultCompactInterval
:
Int
Attributes
protected
Definition Classes
HiveAcidSinkLog
→ CompactibleFileStreamLog
def
deserialize
(
in:
InputStream
)
:
Array
[
HiveAcidSinkStatus
]
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog
final
def
eq
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
def
equals
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
val
fileCleanupDelayMs
:
Long
Attributes
protected
Definition Classes
HiveAcidSinkLog
→ CompactibleFileStreamLog
val
fileManager
:
CheckpointFileManager
Attributes
protected
Definition Classes
HDFSMetadataLog
def
filterInBatch
(
batchId:
Long
)
(
predicate: (
HiveAcidSinkStatus
) ⇒
Boolean
)
:
Option
[
Array
[
HiveAcidSinkStatus
]]
Definition Classes
CompactibleFileStreamLog
def
finalize
()
:
Unit
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
classOf[java.lang.Throwable]
)
def
foreachInBatch
(
batchId:
Long
)
(
fn: (
HiveAcidSinkStatus
) ⇒
Unit
)
:
Unit
Definition Classes
CompactibleFileStreamLog
def
get
(
startId:
Option
[
Long
]
,
endId:
Option
[
Long
]
)
:
Array
[(
Long
,
Array
[
HiveAcidSinkStatus
])]
Definition Classes
HDFSMetadataLog → MetadataLog
def
get
(
batchId:
Long
)
:
Option
[
Array
[
HiveAcidSinkStatus
]]
Definition Classes
HDFSMetadataLog → MetadataLog
final
def
getClass
()
:
Class
[_]
Definition Classes
AnyRef → Any
def
getLatest
()
:
Option
[(
Long
,
Array
[
HiveAcidSinkStatus
])]
Definition Classes
HDFSMetadataLog → MetadataLog
def
getLatestBatchId
()
:
Option
[
Long
]
Definition Classes
HDFSMetadataLog
def
getOrderedBatchFiles
()
:
Array
[
FileStatus
]
Definition Classes
HDFSMetadataLog
def
hashCode
()
:
Int
Definition Classes
AnyRef → Any
def
initializeLogIfNecessary
(
isInterpreter:
Boolean
,
silent:
Boolean
)
:
Boolean
Attributes
protected
Definition Classes
Logging
def
initializeLogIfNecessary
(
isInterpreter:
Boolean
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
isBatchFile
(
path:
Path
)
:
Boolean
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog
val
isDeletingExpiredLog
:
Boolean
Attributes
protected
Definition Classes
HiveAcidSinkLog
→ CompactibleFileStreamLog
final
def
isInstanceOf
[
T0
]
:
Boolean
Definition Classes
Any
def
isTraceEnabled
()
:
Boolean
Attributes
protected
Definition Classes
Logging
def
log
:
Logger
Attributes
protected
Definition Classes
Logging
def
logDebug
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logDebug
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logError
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logError
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logInfo
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logInfo
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logName
:
String
Attributes
protected
Definition Classes
Logging
def
logTrace
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logTrace
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logWarning
(
msg: ⇒
String
,
throwable:
Throwable
)
:
Unit
Attributes
protected
Definition Classes
Logging
def
logWarning
(
msg: ⇒
String
)
:
Unit
Attributes
protected
Definition Classes
Logging
val
metadataPath
:
Path
Definition Classes
HDFSMetadataLog
val
minBatchesToRetain
:
Int
Attributes
protected
Definition Classes
HiveAcidSinkLog
→ CompactibleFileStreamLog
final
def
ne
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
final
def
notify
()
:
Unit
Definition Classes
AnyRef
final
def
notifyAll
()
:
Unit
Definition Classes
AnyRef
def
pathToBatchId
(
path:
Path
)
:
Long
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog
def
purge
(
thresholdBatchId:
Long
)
:
Unit
Definition Classes
HDFSMetadataLog → MetadataLog
def
purgeAfter
(
thresholdBatchId:
Long
)
:
Unit
Definition Classes
HDFSMetadataLog
def
serialize
(
logData:
Array
[
HiveAcidSinkStatus
]
,
out:
OutputStream
)
:
Unit
Definition Classes
CompactibleFileStreamLog → HDFSMetadataLog
def
shouldRetain
(
log:
HiveAcidSinkStatus
)
:
Boolean
Definition Classes
HiveAcidSinkLog
→ CompactibleFileStreamLog
final
def
synchronized
[
T0
]
(
arg0: ⇒
T0
)
:
T0
Definition Classes
AnyRef
def
toString
()
:
String
Definition Classes
AnyRef → Any
final
def
wait
()
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
,
arg1:
Int
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
Inherited from
CompactibleFileStreamLog
[
HiveAcidSinkStatus
]
Inherited from
HDFSMetadataLog
[
Array
[
HiveAcidSinkStatus
]]
Inherited from
Logging
Inherited from
MetadataLog
[
Array
[
HiveAcidSinkStatus
]]
Inherited from
AnyRef
Inherited from
Any
Ungrouped