Replication Arguments for Iceberg Replication policies
| name | data type | constraints | description |
|---|---|---|---|
| sourceIcebergReplicationService | ApiServiceRef | ||
| policyName | string | ||
| includedTableFilters | array of ApiHiveTable | ||
| excludedTableFilters | array of ApiHiveTable | ||
| batchSize | number | ||
| proxyName | string | ||
| exportYarnOptions | IcebergReplicationYarnOptions | ||
| xferYarnOptions | IcebergReplicationYarnOptions | ||
| syncYarnOptions | IcebergReplicationYarnOptions | ||
| shouldReplicateColumnStats | boolean | ||
| atlasReplicationNeeded | boolean | required | |
| atlasChangeMarker | string | ||
| safetyValvesSource | map of IcebergReplicationSafetyValve | ||
| safetyValvesTarget | map of IcebergReplicationSafetyValve | ||
| runDistcpOnSource | boolean | ||
| altStagingLocation | string | ||
| forceTest | boolean | required | |
| alternateTargetDataRoot | string | ||
| dataFileMappingStrategy | DataFileMappingStrategy | ||
| jvmOptsForExport | string | ||
| jvmOptsForXfer | string | ||
| jvmOptsForSync | string | ||
| locationMapping | array of ApiMapEntry |
Example
{
"sourceIcebergReplicationService" : {
"peerName" : "...",
"clusterName" : "...",
"clusterDisplayName" : "...",
"serviceName" : "...",
"serviceDisplayName" : "...",
"serviceType" : "..."
},
"policyName" : "...",
"includedTableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"excludedTableFilters" : [ {
"database" : "...",
"tableName" : "..."
}, {
"database" : "...",
"tableName" : "..."
} ],
"batchSize" : 12345,
"proxyName" : "...",
"exportYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
},
"xferYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
},
"syncYarnOptions" : {
"schedulerPoolName" : "...",
"numberOfMappers" : 12345,
"bandwidthPerMap" : 12345
},
"shouldReplicateColumnStats" : true,
"atlasReplicationNeeded" : true,
"atlasChangeMarker" : "...",
"safetyValvesSource" : {
"property1" : {
"configs" : [ {
"key" : "...",
"value" : "..."
}, {
"key" : "...",
"value" : "..."
} ]
},
"property2" : {
"configs" : [ {
"key" : "...",
"value" : "..."
}, {
"key" : "...",
"value" : "..."
} ]
}
},
"safetyValvesTarget" : {
"property1" : {
"configs" : [ {
"key" : "...",
"value" : "..."
}, {
"key" : "...",
"value" : "..."
} ]
},
"property2" : {
"configs" : [ {
"key" : "...",
"value" : "..."
}, {
"key" : "...",
"value" : "..."
} ]
}
},
"runDistcpOnSource" : true,
"altStagingLocation" : "...",
"forceTest" : true,
"alternateTargetDataRoot" : "...",
"dataFileMappingStrategy" : "RETAIN",
"jvmOptsForExport" : "...",
"jvmOptsForXfer" : "...",
"jvmOptsForSync" : "...",
"locationMapping" : [ {
"key" : "...",
"value" : "..."
}, {
"key" : "...",
"value" : "..."
} ]
}