A replication job schedule.
Replication jobs have service-specific arguments. This object has methods to retrieve arguments for all supported types of replication, but only one argument type is allowed to be set; the backend will check that the provided argument matches the service type where the replication is being scheduled. The replication job's arguments should match the underlying service. Refer to each property's documentation to find out which properties correspond to which services.name | data type | description |
---|---|---|
hdfsArguments | ApiHdfsReplicationArguments | Optional arguments for HDFS replication commands. |
hiveArguments | ApiHiveReplicationArguments | Optional arguments for Hive replication commands. |
hdfsCloudArguments | ApiHdfsCloudReplicationArguments | Optional arguments for HDFS cloud replication commands. |
ozoneReplicationArguments | ApiOzoneReplicationArguments | Optional arguments for HDFS cloud replication commands. |
history | array of ApiReplicationCommand | List of active and/or finished commands for this schedule. |
active | boolean | Read-only field that is true if this schedule is currently active, false if not. Available since API v11. |
hiveCloudArguments | ApiHiveCloudReplicationArguments | Optional arguments for Hive cloud replication commands. |
hbaseArguments | ApiHBaseReplicationArguments | Optional arguments for HBase replication commands. |
hive3Arguments | ApiHive3ReplicationArguments | arguments for Hive3 schedules |
rangerArguments | ApiRangerReplicationArgs | Optional arguments for Ranger replication. |
icebergArguments | ApiIcebergReplicationArguments | Arguments for Iceberg replication. |
Properties inherited from ApiSchedule |
||
id | number | The schedule id. |
displayName | string | The schedule display name. |
description | string | The schedule description. |
startTime | string | The time at which the scheduled activity is triggered for the first time. |
endTime | string | The time after which the scheduled activity will no longer be triggered. |
interval | number | The duration between consecutive triggers of a scheduled activity. Defaults to 0. |
intervalUnit | ApiScheduleInterval | The unit for the repeat interval. |
nextRun | string | Readonly. The time the scheduled command will run next. |
paused | boolean | The paused state for the schedule. The scheduled activity will not be triggered as long as the scheduled is paused. Defaults to false. |
alertOnStart | boolean | Whether to alert on start of the scheduled activity. Defaults to false. |
alertOnSuccess | boolean | Whether to alert on successful completion of the scheduled activity. Defaults to false. |
alertOnFail | boolean | Whether to alert on failure of the scheduled activity. Defaults to false. |
alertOnAbort | boolean | Whether to alert on abort of the scheduled activity. Defaults to false. |
Example
{ "hdfsArguments" : { "sourceService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "STATIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." }, "hiveArguments" : { "sourceService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "tableFilters" : [ { "database" : "...", "tableName" : "..." }, { "database" : "...", "tableName" : "..." } ], "exportDir" : "...", "force" : true, "replicateData" : true, "hdfsArguments" : { "sourceService" : { }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "DYNAMIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." }, "replicateImpalaMetadata" : true, "runInvalidateMetadata" : true, "dryRun" : true, "numThreads" : 12345, "sentryMigration" : true, "skipUrlPermissions" : true, "sentryExportProperties" : { "property1" : "...", "property2" : "..." }, "rangerImportProperties" : { "property1" : "...", "property2" : "..." } }, "hdfsCloudArguments" : { "sourceAccount" : "...", "destinationAccount" : "...", "sourceService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "DYNAMIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." }, "ozoneReplicationArguments" : { "originalSourcePath" : "...", "originalDestinationPath" : "...", "replicationType" : "FULL_FILE_LISTING", "sourceService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "STATIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." }, "history" : [ { "hdfsResult" : { "progress" : 12345, "throughput" : 12345.0, "remainingTime" : 12345, "estimatedCompletionTime" : "...", "counters" : [ { }, { } ], "numFilesDryRun" : 12345, "numBytesDryRun" : 12345, "numFilesExpected" : 12345, "numBytesExpected" : 12345, "numFilesCopied" : 12345, "numBytesCopied" : 12345, "numFilesSkipped" : 12345, "numBytesSkipped" : 12345, "numFilesDeleted" : 12345, "numFilesCopyFailed" : 12345, "numBytesCopyFailed" : 12345, "setupError" : "...", "jobId" : "...", "jobDetailsUri" : "...", "dryRun" : true, "snapshottedDirs" : [ "...", "..." ], "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "failedFiles" : [ "...", "..." ] }, "ozoneResult" : { "commandResultMessage" : "...", "replicationType" : "INCREMENTAL_WITH_FALLBACK_TO_FULL_FILE_LISTING", "replicationCommandType" : "FULL_FILE_LISTING", "progress" : 12345, "throughput" : 12345.0, "remainingTime" : 12345, "estimatedCompletionTime" : "...", "counters" : [ { }, { } ], "numFilesDryRun" : 12345, "numBytesDryRun" : 12345, "numFilesExpected" : 12345, "numBytesExpected" : 12345, "numFilesCopied" : 12345, "numBytesCopied" : 12345, "numFilesSkipped" : 12345, "numBytesSkipped" : 12345, "numFilesDeleted" : 12345, "numFilesCopyFailed" : 12345, "numBytesCopyFailed" : 12345, "setupError" : "...", "jobId" : "...", "jobDetailsUri" : "...", "dryRun" : true, "snapshottedDirs" : [ "...", "..." ], "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "failedFiles" : [ "...", "..." ] }, "hiveResult" : { "phase" : "...", "tableCount" : 12345, "tables" : [ { }, { } ], "impalaUDFCount" : 12345, "hiveUDFCount" : 12345, "impalaUDFs" : [ { }, { } ], "hiveUDFs" : [ { }, { } ], "errorCount" : 12345, "errors" : [ { }, { } ], "dataReplicationResult" : { }, "dryRun" : true, "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "directoryForMetadata" : "...", "statsAvailable" : true, "dbProcessed" : 12345, "tableProcessed" : 12345, "partitionProcessed" : 12345, "functionProcessed" : 12345, "indexProcessed" : 12345, "statsProcessed" : 12345, "dbExpected" : 12345, "tableExpected" : 12345, "partitionExpected" : 12345, "functionExpected" : 12345, "indexExpected" : 12345, "statsExpected" : 12345 }, "hiveOnTezResult" : { "type" : "...", "status" : "...", "error" : "...", "tablesCurrent" : 12345, "tablesTotal" : 12345, "functionsCurrent" : 12345, "functionsTotal" : 12345, "eventsCurrent" : 12345, "eventsTotal" : 12345, "policiesCurrent" : 12345, "policiesTotal" : 12345, "entitiesCurrent" : 12345, "entitiesTotal" : 12345, "origin" : "TARGET" }, "hbaseInfo" : { "replicationOperations" : [ "...", "..." ] }, "rangerResult" : { "total" : 12345, "created" : 12345, "skipped" : 12345, "failed" : 12345, "skippedDueToTimeout" : 12345 }, "icebergResult" : { "numberOfTablesProcessed" : 12345, "numberOfFilesCopied" : 12345, "numberOfFilesDeleted" : 12345, "numberOfManifestsTransformed" : 12345, "failed" : true, "timestamp" : "...", "copyMetadataJobDetailsUri" : "...", "copyDataJobDetailsUri" : "...", "copyDeleteDataJobDetailsUri" : "...", "copyMetadataJobId" : "...", "copyDataJobId" : "...", "copyDeleteDataJobId" : "..." }, "id" : 12345, "name" : "...", "startTime" : "...", "endTime" : "...", "active" : true, "success" : true, "resultMessage" : "...", "resultDataUrl" : "...", "clusterRef" : { "clusterName" : "...", "displayName" : "..." }, "serviceRef" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "roleRef" : { "clusterName" : "...", "serviceName" : "...", "roleName" : "...", "healthSummary" : "DISABLED", "roleStatus" : "HISTORY_NOT_AVAILABLE" }, "hostRef" : { "hostId" : "...", "hostname" : "..." }, "parent" : { "id" : 12345, "name" : "...", "startTime" : "...", "endTime" : "...", "active" : true, "success" : true, "resultMessage" : "...", "resultDataUrl" : "...", "clusterRef" : { }, "serviceRef" : { }, "roleRef" : { }, "hostRef" : { }, "parent" : { }, "children" : { }, "canRetry" : true }, "children" : { "items" : [ { }, { } ] }, "canRetry" : true }, { "hdfsResult" : { "progress" : 12345, "throughput" : 12345.0, "remainingTime" : 12345, "estimatedCompletionTime" : "...", "counters" : [ { }, { } ], "numFilesDryRun" : 12345, "numBytesDryRun" : 12345, "numFilesExpected" : 12345, "numBytesExpected" : 12345, "numFilesCopied" : 12345, "numBytesCopied" : 12345, "numFilesSkipped" : 12345, "numBytesSkipped" : 12345, "numFilesDeleted" : 12345, "numFilesCopyFailed" : 12345, "numBytesCopyFailed" : 12345, "setupError" : "...", "jobId" : "...", "jobDetailsUri" : "...", "dryRun" : true, "snapshottedDirs" : [ "...", "..." ], "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "failedFiles" : [ "...", "..." ] }, "ozoneResult" : { "commandResultMessage" : "...", "replicationType" : "INCREMENTAL_ONLY", "replicationCommandType" : "FULL_FILE_LISTING", "progress" : 12345, "throughput" : 12345.0, "remainingTime" : 12345, "estimatedCompletionTime" : "...", "counters" : [ { }, { } ], "numFilesDryRun" : 12345, "numBytesDryRun" : 12345, "numFilesExpected" : 12345, "numBytesExpected" : 12345, "numFilesCopied" : 12345, "numBytesCopied" : 12345, "numFilesSkipped" : 12345, "numBytesSkipped" : 12345, "numFilesDeleted" : 12345, "numFilesCopyFailed" : 12345, "numBytesCopyFailed" : 12345, "setupError" : "...", "jobId" : "...", "jobDetailsUri" : "...", "dryRun" : true, "snapshottedDirs" : [ "...", "..." ], "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "failedFiles" : [ "...", "..." ] }, "hiveResult" : { "phase" : "...", "tableCount" : 12345, "tables" : [ { }, { } ], "impalaUDFCount" : 12345, "hiveUDFCount" : 12345, "impalaUDFs" : [ { }, { } ], "hiveUDFs" : [ { }, { } ], "errorCount" : 12345, "errors" : [ { }, { } ], "dataReplicationResult" : { }, "dryRun" : true, "runAsUser" : "...", "runOnSourceAsUser" : "...", "logPath" : "...", "directoryForMetadata" : "...", "statsAvailable" : true, "dbProcessed" : 12345, "tableProcessed" : 12345, "partitionProcessed" : 12345, "functionProcessed" : 12345, "indexProcessed" : 12345, "statsProcessed" : 12345, "dbExpected" : 12345, "tableExpected" : 12345, "partitionExpected" : 12345, "functionExpected" : 12345, "indexExpected" : 12345, "statsExpected" : 12345 }, "hiveOnTezResult" : { "type" : "...", "status" : "...", "error" : "...", "tablesCurrent" : 12345, "tablesTotal" : 12345, "functionsCurrent" : 12345, "functionsTotal" : 12345, "eventsCurrent" : 12345, "eventsTotal" : 12345, "policiesCurrent" : 12345, "policiesTotal" : 12345, "entitiesCurrent" : 12345, "entitiesTotal" : 12345, "origin" : "SOURCE" }, "hbaseInfo" : { "replicationOperations" : [ "...", "..." ] }, "rangerResult" : { "total" : 12345, "created" : 12345, "skipped" : 12345, "failed" : 12345, "skippedDueToTimeout" : 12345 }, "icebergResult" : { "numberOfTablesProcessed" : 12345, "numberOfFilesCopied" : 12345, "numberOfFilesDeleted" : 12345, "numberOfManifestsTransformed" : 12345, "failed" : true, "timestamp" : "...", "copyMetadataJobDetailsUri" : "...", "copyDataJobDetailsUri" : "...", "copyDeleteDataJobDetailsUri" : "...", "copyMetadataJobId" : "...", "copyDataJobId" : "...", "copyDeleteDataJobId" : "..." }, "id" : 12345, "name" : "...", "startTime" : "...", "endTime" : "...", "active" : true, "success" : true, "resultMessage" : "...", "resultDataUrl" : "...", "clusterRef" : { "clusterName" : "...", "displayName" : "..." }, "serviceRef" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "roleRef" : { "clusterName" : "...", "serviceName" : "...", "roleName" : "...", "healthSummary" : "DISABLED", "roleStatus" : "BUSY" }, "hostRef" : { "hostId" : "...", "hostname" : "..." }, "parent" : { "id" : 12345, "name" : "...", "startTime" : "...", "endTime" : "...", "active" : true, "success" : true, "resultMessage" : "...", "resultDataUrl" : "...", "clusterRef" : { }, "serviceRef" : { }, "roleRef" : { }, "hostRef" : { }, "parent" : { }, "children" : { }, "canRetry" : true }, "children" : { "items" : [ { }, { } ] }, "canRetry" : true } ], "active" : true, "hiveCloudArguments" : { "sourceAccount" : "...", "destinationAccount" : "...", "cloudRootPath" : "...", "replicationOption" : "METADATA_AND_DATA", "sourceService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "tableFilters" : [ { "database" : "...", "tableName" : "..." }, { "database" : "...", "tableName" : "..." } ], "exportDir" : "...", "force" : true, "replicateData" : true, "hdfsArguments" : { "sourceService" : { }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "STATIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." }, "replicateImpalaMetadata" : true, "runInvalidateMetadata" : true, "dryRun" : true, "numThreads" : 12345, "sentryMigration" : true, "skipUrlPermissions" : true, "sentryExportProperties" : { "property1" : "...", "property2" : "..." }, "rangerImportProperties" : { "property1" : "...", "property2" : "..." } }, "hbaseArguments" : { "sourceHBaseService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "peerState" : "DISABLED", "hbasePeerId" : "...", "hbaseClusterKey" : "...", "endPointClassName" : "...", "tables" : [ { "tableName" : "...", "initialSnapshot" : true, "snapshotSucceeded" : true }, { "tableName" : "...", "initialSnapshot" : true, "snapshotSucceeded" : true } ], "database" : { "failedSnapshotTables" : [ "...", "..." ], "initialSnapshot" : true, "replicationStrategy" : "ALL_TABLES" }, "replicationProperties" : { "property1" : "...", "property2" : "..." }, "sourceAccount" : "...", "sourceSchedulerPool" : "...", "numMappers" : 12345, "bandwidthPerMap" : 12345, "snapshotScratchLocation" : "...", "validateReplicationSetup" : true, "maxParallelSnapshots" : 12345 }, "hive3Arguments" : { "sourceHiveService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "status" : "ENABLED", "rangerReplication" : true, "atlasReplication" : true, "externalTableReplication" : true, "externalTableBaseDir" : "...", "distcpOnTarget" : true, "numMaps" : 12345, "bandwidthPerMap" : 12345, "policyOptions" : { "property1" : "...", "property2" : "..." }, "sourceDbName" : "...", "targetDbName" : "...", "policyName" : "...", "scheduleClause" : "...", "runAs" : "...", "hiveOp" : "...", "hiveUpdateOp" : "...", "excludeSource" : true, "excludeTarget" : true, "failoverStatus" : "...", "cloudArgs" : { "sourceAccount" : "..." }, "remoteDbParams" : "..." }, "rangerArguments" : { "sourceRangerService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "exportArgs" : { "rangerServiceNames" : [ "...", "..." ] }, "transformArgs" : { "rangerServiceNamesMapping" : { "property1" : "...", "property2" : "..." }, "rangerUsersMapping" : { "property1" : "...", "property2" : "..." }, "rangerResourcesMapping" : { "property1" : "...", "property2" : "..." } }, "importArgs" : { "importStrategy" : "OVERRIDE" }, "auditLogDistCpArgs" : { "sourceService" : { }, "sourcePath" : "...", "destinationPath" : "...", "mapreduceServiceName" : "...", "schedulerPoolName" : "...", "userName" : "...", "sourceUser" : "...", "numMaps" : 12345, "dryRun" : true, "bandwidthPerMap" : 12345, "abortOnError" : true, "removeMissingFiles" : true, "preserveReplicationCount" : true, "preserveBlockSize" : true, "preservePermissions" : true, "logPath" : "...", "skipChecksumChecks" : true, "skipListingChecksumChecks" : true, "skipTrash" : true, "replicationStrategy" : "STATIC", "preserveXAttrs" : true, "exclusionFilters" : [ "...", "..." ], "raiseSnapshotDiffFailures" : true, "deleteLatestSourceSnapshotOnJobFailure" : true, "numFetchThreads" : 12345, "destinationCloudAccount" : "..." } }, "icebergArguments" : { "sourceIcebergReplicationService" : { "peerName" : "...", "clusterName" : "...", "serviceName" : "...", "serviceDisplayName" : "...", "serviceType" : "..." }, "policyName" : "...", "includedTableFilters" : [ { "database" : "...", "tableName" : "..." }, { "database" : "...", "tableName" : "..." } ], "excludedTableFilters" : [ { "database" : "...", "tableName" : "..." }, { "database" : "...", "tableName" : "..." } ], "batchSize" : 12345, "proxyName" : "...", "exportYarnOptions" : { "schedulerPoolName" : "...", "numberOfMappers" : 12345, "bandwidthPerMap" : 12345 }, "xferYarnOptions" : { "schedulerPoolName" : "...", "numberOfMappers" : 12345, "bandwidthPerMap" : 12345 }, "syncYarnOptions" : { "schedulerPoolName" : "...", "numberOfMappers" : 12345, "bandwidthPerMap" : 12345 }, "shouldReplicateColumnStats" : true, "safetyValvesSource" : { "property1" : { "configs" : [ { }, { } ] }, "property2" : { "configs" : [ { }, { } ] } }, "safetyValvesTarget" : { "property1" : { "configs" : [ { }, { } ] }, "property2" : { "configs" : [ { }, { } ] } } }, "id" : 12345, "displayName" : "...", "description" : "...", "startTime" : "...", "endTime" : "...", "interval" : 12345, "intervalUnit" : "DAY", "nextRun" : "...", "paused" : true, "alertOnStart" : true, "alertOnSuccess" : true, "alertOnFail" : true, "alertOnAbort" : true }