A replication job schedule.
Replication jobs have service-specific arguments. This object has methods to retrieve arguments for all supported types of replication, but only one argument type is allowed to be set; the backend will check that the provided argument matches the service type where the replication is being scheduled. The replication job's arguments should match the underlying service. Refer to each property's documentation to find out which properties correspond to which services.| name | data type | description |
|---|---|---|
| hdfsArguments | ApiHdfsReplicationArguments | Optional arguments for HDFS replication commands. |
| hiveArguments | ApiHiveReplicationArguments | Optional arguments for Hive replication commands. |
| hdfsCloudArguments | ApiHdfsCloudReplicationArguments | Optional arguments for HDFS cloud replication commands. |
| ozoneReplicationArguments | ApiOzoneReplicationArguments | Optional arguments for HDFS cloud replication commands. |
| history | array of ApiReplicationCommand | List of active and/or finished commands for this schedule. |
| active | boolean | Read-only field that is true if this schedule is currently active, false if not. Available since API v11. |
| hiveCloudArguments | ApiHiveCloudReplicationArguments | Optional arguments for Hive cloud replication commands. |
| hbaseArguments | ApiHBaseReplicationArguments | Optional arguments for HBase replication commands. |
| hive3Arguments | ApiHive3ReplicationArguments | arguments for Hive3 schedules |
| rangerArguments | ApiRangerReplicationArgs | Optional arguments for Ranger replication. |
| icebergArguments | ApiIcebergReplicationArguments | Arguments for Iceberg replication. |
Properties inherited from ApiSchedule |
||
| id | number | The schedule id. |
| displayName | string | The schedule display name. |
| description | string | The schedule description. |
| startTime | string | The time at which the scheduled activity is triggered for the first time. |
| endTime | string | The time after which the scheduled activity will no longer be triggered. |
| interval | number | The duration between consecutive triggers of a scheduled activity. Defaults to 0. |
| intervalUnit | ApiScheduleInterval | The unit for the repeat interval. |
| nextRun | string | Readonly. The time the scheduled command will run next. |
| paused | boolean | The paused state for the schedule. The scheduled activity will not be triggered as long as the scheduled is paused. Defaults to false. |
| alertOnStart | boolean | Whether to alert on start of the scheduled activity. Defaults to false. |
| alertOnSuccess | boolean | Whether to alert on successful completion of the scheduled activity. Defaults to false. |
| alertOnFail | boolean | Whether to alert on failure of the scheduled activity. Defaults to false. |
| alertOnAbort | boolean | Whether to alert on abort of the scheduled activity. Defaults to false. |
Example
{
"hdfsArguments" : {
"sourceService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "STATIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
},
"hiveArguments" : {
"sourceService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"tableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"exportDir" : "...",
"force" : true,
"replicateData" : true,
"hdfsArguments" : {
"sourceService" : { },
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "DYNAMIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
},
"replicateImpalaMetadata" : true,
"runInvalidateMetadata" : true,
"dryRun" : true,
"numThreads" : 12345,
"sentryMigration" : true,
"skipUrlPermissions" : true
},
"hdfsCloudArguments" : {
"sourceAccount" : "...",
"destinationAccount" : "...",
"sourceService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "STATIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
},
"ozoneReplicationArguments" : {
"originalSourcePath" : "...",
"originalDestinationPath" : "...",
"replicationType" : "FULL_FILE_LISTING",
"sourceService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "DYNAMIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
},
"history" : [ {
"hdfsResult" : {
"progress" : 12345,
"throughput" : 12345.0,
"remainingTime" : 12345,
"estimatedCompletionTime" : "...",
"counters" : [ { }, { } ],
"numFilesDryRun" : 12345,
"numBytesDryRun" : 12345,
"numFilesExpected" : 12345,
"numBytesExpected" : 12345,
"numFilesCopied" : 12345,
"numBytesCopied" : 12345,
"numFilesSkipped" : 12345,
"numBytesSkipped" : 12345,
"numFilesDeleted" : 12345,
"numFilesCopyFailed" : 12345,
"numBytesCopyFailed" : 12345,
"setupError" : "...",
"jobId" : "...",
"jobDetailsUri" : "...",
"dryRun" : true,
"snapshottedDirs" : [ "...", "..." ],
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"failedFiles" : [ "...", "..." ]
},
"ozoneResult" : {
"replicationType" : "INCREMENTAL_WITH_FALLBACK_TO_FULL_FILE_LISTING",
"replicationCommandType" : "INCREMENTAL",
"progress" : 12345,
"throughput" : 12345.0,
"remainingTime" : 12345,
"estimatedCompletionTime" : "...",
"counters" : [ { }, { } ],
"numFilesDryRun" : 12345,
"numBytesDryRun" : 12345,
"numFilesExpected" : 12345,
"numBytesExpected" : 12345,
"numFilesCopied" : 12345,
"numBytesCopied" : 12345,
"numFilesSkipped" : 12345,
"numBytesSkipped" : 12345,
"numFilesDeleted" : 12345,
"numFilesCopyFailed" : 12345,
"numBytesCopyFailed" : 12345,
"setupError" : "...",
"jobId" : "...",
"jobDetailsUri" : "...",
"dryRun" : true,
"snapshottedDirs" : [ "...", "..." ],
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"failedFiles" : [ "...", "..." ]
},
"hiveResult" : {
"phase" : "...",
"tableCount" : 12345,
"tables" : [ { }, { } ],
"impalaUDFCount" : 12345,
"hiveUDFCount" : 12345,
"impalaUDFs" : [ { }, { } ],
"hiveUDFs" : [ { }, { } ],
"errorCount" : 12345,
"errors" : [ { }, { } ],
"dataReplicationResult" : { },
"dryRun" : true,
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"directoryForMetadata" : "...",
"statsAvailable" : true,
"dbProcessed" : 12345,
"tableProcessed" : 12345,
"partitionProcessed" : 12345,
"functionProcessed" : 12345,
"indexProcessed" : 12345,
"statsProcessed" : 12345,
"dbExpected" : 12345,
"tableExpected" : 12345,
"partitionExpected" : 12345,
"functionExpected" : 12345,
"indexExpected" : 12345,
"statsExpected" : 12345
},
"hiveOnTezResult" : {
"type" : "...",
"status" : "...",
"error" : "...",
"tablesCurrent" : 12345,
"tablesTotal" : 12345,
"functionsCurrent" : 12345,
"functionsTotal" : 12345,
"eventsCurrent" : 12345,
"eventsTotal" : 12345,
"policiesCurrent" : 12345,
"policiesTotal" : 12345,
"entitiesCurrent" : 12345,
"entitiesTotal" : 12345,
"origin" : "TARGET"
},
"hbaseInfo" : {
"replicationOperations" : [ "...", "..." ]
},
"rangerResult" : {
"total" : 12345,
"created" : 12345,
"skipped" : 12345,
"failed" : 12345,
"skippedDueToTimeout" : 12345
},
"icebergResult" : {
"numberOfTablesProcessed" : 12345,
"numberOfFilesCopied" : 12345,
"numberOfFilesDeleted" : 12345,
"numberOfManifestsTransformed" : 12345,
"failed" : true,
"timestamp" : "..."
},
"id" : 12345,
"name" : "...",
"startTime" : "...",
"endTime" : "...",
"active" : true,
"success" : true,
"resultMessage" : "...",
"resultDataUrl" : "...",
"clusterRef" : {
"clusterName" : "...",
"displayName" : "..."
},
"serviceRef" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"roleRef" : {
"clusterName" : "...",
"serviceName" : "...",
"roleName" : "...",
"healthSummary" : "BAD",
"roleStatus" : "BUSY"
},
"hostRef" : {
"hostId" : "...",
"hostname" : "..."
},
"parent" : {
"id" : 12345,
"name" : "...",
"startTime" : "...",
"endTime" : "...",
"active" : true,
"success" : true,
"resultMessage" : "...",
"resultDataUrl" : "...",
"clusterRef" : { },
"serviceRef" : { },
"roleRef" : { },
"hostRef" : { },
"parent" : { },
"children" : { },
"canRetry" : true
},
"children" : {
"items" : [ { }, { } ]
},
"canRetry" : true
}, {
"hdfsResult" : {
"progress" : 12345,
"throughput" : 12345.0,
"remainingTime" : 12345,
"estimatedCompletionTime" : "...",
"counters" : [ { }, { } ],
"numFilesDryRun" : 12345,
"numBytesDryRun" : 12345,
"numFilesExpected" : 12345,
"numBytesExpected" : 12345,
"numFilesCopied" : 12345,
"numBytesCopied" : 12345,
"numFilesSkipped" : 12345,
"numBytesSkipped" : 12345,
"numFilesDeleted" : 12345,
"numFilesCopyFailed" : 12345,
"numBytesCopyFailed" : 12345,
"setupError" : "...",
"jobId" : "...",
"jobDetailsUri" : "...",
"dryRun" : true,
"snapshottedDirs" : [ "...", "..." ],
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"failedFiles" : [ "...", "..." ]
},
"ozoneResult" : {
"replicationType" : "INCREMENTAL_ONLY",
"replicationCommandType" : "FULL_FILE_LISTING_FALLBACK",
"progress" : 12345,
"throughput" : 12345.0,
"remainingTime" : 12345,
"estimatedCompletionTime" : "...",
"counters" : [ { }, { } ],
"numFilesDryRun" : 12345,
"numBytesDryRun" : 12345,
"numFilesExpected" : 12345,
"numBytesExpected" : 12345,
"numFilesCopied" : 12345,
"numBytesCopied" : 12345,
"numFilesSkipped" : 12345,
"numBytesSkipped" : 12345,
"numFilesDeleted" : 12345,
"numFilesCopyFailed" : 12345,
"numBytesCopyFailed" : 12345,
"setupError" : "...",
"jobId" : "...",
"jobDetailsUri" : "...",
"dryRun" : true,
"snapshottedDirs" : [ "...", "..." ],
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"failedFiles" : [ "...", "..." ]
},
"hiveResult" : {
"phase" : "...",
"tableCount" : 12345,
"tables" : [ { }, { } ],
"impalaUDFCount" : 12345,
"hiveUDFCount" : 12345,
"impalaUDFs" : [ { }, { } ],
"hiveUDFs" : [ { }, { } ],
"errorCount" : 12345,
"errors" : [ { }, { } ],
"dataReplicationResult" : { },
"dryRun" : true,
"runAsUser" : "...",
"runOnSourceAsUser" : "...",
"logPath" : "...",
"directoryForMetadata" : "...",
"statsAvailable" : true,
"dbProcessed" : 12345,
"tableProcessed" : 12345,
"partitionProcessed" : 12345,
"functionProcessed" : 12345,
"indexProcessed" : 12345,
"statsProcessed" : 12345,
"dbExpected" : 12345,
"tableExpected" : 12345,
"partitionExpected" : 12345,
"functionExpected" : 12345,
"indexExpected" : 12345,
"statsExpected" : 12345
},
"hiveOnTezResult" : {
"type" : "...",
"status" : "...",
"error" : "...",
"tablesCurrent" : 12345,
"tablesTotal" : 12345,
"functionsCurrent" : 12345,
"functionsTotal" : 12345,
"eventsCurrent" : 12345,
"eventsTotal" : 12345,
"policiesCurrent" : 12345,
"policiesTotal" : 12345,
"entitiesCurrent" : 12345,
"entitiesTotal" : 12345,
"origin" : "TARGET"
},
"hbaseInfo" : {
"replicationOperations" : [ "...", "..." ]
},
"rangerResult" : {
"total" : 12345,
"created" : 12345,
"skipped" : 12345,
"failed" : 12345,
"skippedDueToTimeout" : 12345
},
"icebergResult" : {
"numberOfTablesProcessed" : 12345,
"numberOfFilesCopied" : 12345,
"numberOfFilesDeleted" : 12345,
"numberOfManifestsTransformed" : 12345,
"failed" : true,
"timestamp" : "..."
},
"id" : 12345,
"name" : "...",
"startTime" : "...",
"endTime" : "...",
"active" : true,
"success" : true,
"resultMessage" : "...",
"resultDataUrl" : "...",
"clusterRef" : {
"clusterName" : "...",
"displayName" : "..."
},
"serviceRef" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"roleRef" : {
"clusterName" : "...",
"serviceName" : "...",
"roleName" : "...",
"healthSummary" : "HISTORY_NOT_AVAILABLE",
"roleStatus" : "STOPPING"
},
"hostRef" : {
"hostId" : "...",
"hostname" : "..."
},
"parent" : {
"id" : 12345,
"name" : "...",
"startTime" : "...",
"endTime" : "...",
"active" : true,
"success" : true,
"resultMessage" : "...",
"resultDataUrl" : "...",
"clusterRef" : { },
"serviceRef" : { },
"roleRef" : { },
"hostRef" : { },
"parent" : { },
"children" : { },
"canRetry" : true
},
"children" : {
"items" : [ { }, { } ]
},
"canRetry" : true
} ],
"active" : true,
"hiveCloudArguments" : {
"sourceAccount" : "...",
"destinationAccount" : "...",
"cloudRootPath" : "...",
"replicationOption" : "KEEP_DATA_IN_CLOUD",
"sourceService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"tableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"exportDir" : "...",
"force" : true,
"replicateData" : true,
"hdfsArguments" : {
"sourceService" : { },
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "DYNAMIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
},
"replicateImpalaMetadata" : true,
"runInvalidateMetadata" : true,
"dryRun" : true,
"numThreads" : 12345,
"sentryMigration" : true,
"skipUrlPermissions" : true
},
"hbaseArguments" : {
"sourceHBaseService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"peerState" : "DISABLED",
"hbasePeerId" : "...",
"hbaseClusterKey" : "...",
"endPointClassName" : "...",
"tables" : [ {
"tableName" : "...",
"initialSnapshot" : true,
"snapshotSucceeded" : true
}, {
"tableName" : "...",
"initialSnapshot" : true,
"snapshotSucceeded" : true
} ],
"database" : {
"failedSnapshotTables" : [ "...", "..." ],
"initialSnapshot" : true
},
"replicationProperties" : {
"property1" : "...",
"property2" : "..."
},
"sourceAccount" : "...",
"sourceSchedulerPool" : "...",
"numMappers" : 12345,
"snapshotScratchLocation" : "...",
"validateReplicationSetup" : true
},
"hive3Arguments" : {
"sourceHiveService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"status" : "ENABLED",
"rangerReplication" : true,
"atlasReplication" : true,
"externalTableReplication" : true,
"externalTableBaseDir" : "...",
"distcpOnTarget" : true,
"numMaps" : 12345,
"bandwidthPerMap" : 12345,
"policyOptions" : {
"property1" : "...",
"property2" : "..."
},
"sourceDbName" : "...",
"targetDbName" : "...",
"policyName" : "...",
"scheduleClause" : "...",
"runAs" : "...",
"hiveOp" : "...",
"hiveUpdateOp" : "...",
"excludeSource" : true,
"excludeTarget" : true,
"failoverStatus" : "...",
"cloudArgs" : {
"sourceAccount" : "..."
},
"remoteDbParams" : "..."
},
"rangerArguments" : {
"sourceRangerService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"exportArgs" : {
"rangerServiceNames" : [ "...", "..." ]
},
"transformArgs" : {
"rangerServiceNamesMapping" : {
"property1" : "...",
"property2" : "..."
},
"rangerUsersMapping" : {
"property1" : "...",
"property2" : "..."
},
"rangerResourcesMapping" : {
"property1" : "...",
"property2" : "..."
}
},
"importArgs" : {
"importStrategy" : "MERGE"
},
"auditLogDistCpArgs" : {
"sourceService" : { },
"sourcePath" : "...",
"destinationPath" : "...",
"mapreduceServiceName" : "...",
"schedulerPoolName" : "...",
"userName" : "...",
"sourceUser" : "...",
"numMaps" : 12345,
"dryRun" : true,
"bandwidthPerMap" : 12345,
"abortOnError" : true,
"removeMissingFiles" : true,
"preserveReplicationCount" : true,
"preserveBlockSize" : true,
"preservePermissions" : true,
"logPath" : "...",
"skipChecksumChecks" : true,
"skipListingChecksumChecks" : true,
"skipTrash" : true,
"replicationStrategy" : "DYNAMIC",
"preserveXAttrs" : true,
"exclusionFilters" : [ "...", "..." ],
"raiseSnapshotDiffFailures" : true,
"deleteLatestSourceSnapshotOnJobFailure" : true,
"numFetchThreads" : 12345,
"destinationCloudAccount" : "..."
}
},
"icebergArguments" : {
"sourceIcebergReplicationService" : {
"peerName" : "...",
"clusterName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"policyName" : "...",
"includedTableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"excludedTableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"batchSize" : 12345,
"exportYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
},
"xferYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
},
"syncYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
}
},
"id" : 12345,
"displayName" : "...",
"description" : "...",
"startTime" : "...",
"endTime" : "...",
"interval" : 12345,
"intervalUnit" : "DAY",
"nextRun" : "...",
"paused" : true,
"alertOnStart" : true,
"alertOnSuccess" : true,
"alertOnFail" : true,
"alertOnAbort" : true
}