config.yaml.example

# Example configuration for migrating from Cassandra:
source:
  type: cassandra
  host: cassandra-server-01
  port: 9042
  #optional, if not specified None will be used
  #localDC: <localdc>
  #credentials:
  #  username: <user>
  #  password: <pass>
  # SSL as per https://github.com/scylladb/spark-cassandra-connector/blob/master/doc/reference.md#cassandra-ssl-connection-options
  #sslOptions:
  #  clientAuthEnabled: false
  #  enabled: false
  #  all below are optional! (generally just trustStorePassword and trustStorePath is needed)
  #  trustStorePassword: <pass>
  #  trustStorePath: <path>
  #  trustStoreType: JKS
  #  keyStorePassword: <pass>
  #  keyStorePath: <path>
  #  keyStoreType: JKS
  #  enabledAlgorithms:
  #   - TLS_RSA_WITH_AES_128_CBC_SHA
  #   - TLS_RSA_WITH_AES_256_CBC_SHA
  #  protocol: TLS
  keyspace: stocks
  table: stocks
  # Consistency Level for the source connection
  # Options are: LOCAL_ONE, ONE, LOCAL_QUORUM, QUORUM.
  # Connector driver default is LOCAL_ONE. Our recommendation is LOCAL_QUORUM.
  # If using ONE or LOCAL_ONE, ensure the source system is fully repaired.
  consistencyLevel: LOCAL_QUORUM
  # Preserve TTLs and WRITETIMEs of cells in the source database. Note that this
  # option is *incompatible* when copying tables with collections (lists, maps, sets).
  preserveTimestamps: true
  # Number of splits to use - this should be at minimum the amount of cores
  # available in the Spark cluster, and optimally more; higher splits will lead
  # to more fine-grained resumes. Aim for 8 * (Spark cores).
  splitCount: 256
  # Number of connections to use to Cassandra when copying
  connections: 8
  # Number of rows to fetch in each read
  fetchSize: 1000
  # Optional condition to filter source table data that will be migrated
  # where: race_start_date = '2015-05-27' AND race_end_date = '2015-05-27'

  # Optional - Connect via a DataStax Astra Cloud Secure Connect bundle.
  # When this block is present the 'host', 'port', 'localDC' and 'sslOptions' fields above must
  # NOT be set: they are carried by the bundle and would conflict with it. Authentication
  # credentials must still be supplied via the 'credentials' block.
  #
  # Accepted secureBundlePath forms:
  #   - Absolute local path: /opt/migrator/secure-connect-mycluster.zip
  #     (auto-converted to file:// URL at runtime; must exist on every node)
  #   - https:// URL reachable from every node
  #   - Bare filename: secure-connect-mycluster.zip
  #     (for bundles distributed via spark-submit --files; resolved via SparkFiles.get)
  #   - s3:// or s3a:// URL — works in standard Spark environments where Hadoop URL handlers
  #     are registered. For maximum reliability, prefer --files instead:
  #       spark-submit --files s3://bucket/bundle.zip ...
  #     then set secureBundlePath to just the filename: bundle.zip
  #
  # Do not embed credentials or signed-token query strings in the URL.
  #cloud:
  #  secureBundlePath: /opt/migrator/secure-connect-mycluster.zip

# Example for connecting to a DataStax Astra source via a Cloud Secure Connect bundle:
# source:
#   type: cassandra
#   cloud:
#     secureBundlePath: /opt/migrator/secure-connect-mycluster.zip
#   credentials:
#     username: <client-id>
#     password: <client-secret>
#   keyspace: <keyspace>
#   table: <table>
#   consistencyLevel: LOCAL_QUORUM
#   preserveTimestamps: true
#   splitCount: 256
#   connections: 8
#   fetchSize: 1000
#
# Alternative: distribute the bundle via --files (recommended for S3-hosted bundles):
#   spark-submit --files s3://my-bucket/secure-connect-mycluster.zip \
#     --class com.scylladb.migrator.Migrator migrator.jar
# then use a bare filename in the config:
# source:
#   type: cassandra
#   cloud:
#     secureBundlePath: secure-connect-mycluster.zip
#   ...

# Example for loading from MySQL:
# source:
#   type: mysql
#   host: mysql-server
#   port: 3306
#   database: mydb
#   table: mytable
#   credentials:
#     username: <user>
#     password: <pass>
#   # Number of rows to fetch per JDBC round-trip. Defaults to 1000 if omitted.
#   # Use lower values (e.g. 100) for tables with large blobs, higher values
#   # (e.g. 1000+) for small rows.
#   fetchSize: 1000
#   # Optional - columns forming the primary key, required for validation
#   # primaryKey:
#   #   - id
#   # Optional - partition the read across multiple Spark tasks
#   # partitionColumn: id
#   # numPartitions: 8
#   # lowerBound and upperBound must match the partition column's JDBC type.
#   # For numeric columns use integer literals:
#   # lowerBound: 0
#   # upperBound: 1000000
#   # For DATE columns use ISO date literals:
#   # lowerBound: "2024-01-01"
#   # upperBound: "2024-02-01"
#   # For TIMESTAMP/DATETIME columns use timestamp literals:
#   # lowerBound: "2024-01-01 00:00:00"
#   # upperBound: "2024-01-02 00:00:00"
#   # NOTE: Partitioned MySQL reads use multiple independent JDBC statements/connections.
#   # They do not provide one global point-in-time snapshot across all Spark partitions.
#   # If exact consistency matters, quiesce or otherwise freeze the source table first.
#   # Optional - filter rows (treated as trusted SQL input, not sanitized)
#   # where: "status = 'active'"
#   # Optional - explicit zero-date handling. Defaults to EXCEPTION (fail fast).
#   # Setting CONVERT_TO_NULL or ROUND changes source-data semantics and should only
#   # be used when that behavior is intentional.
#   # zeroDateTimeBehavior: CONVERT_TO_NULL
#   # Optional - extra JDBC connection properties (e.g. SSL, timeouts)
#   # connectionProperties:
#   #   connectTimeout: "5000"
#   #   socketTimeout: "30000"
#   #
#   # NOTE: Resume/savepoint support is not available for MySQL migrations.
#   # Spark JDBC reads do not expose durable per-range progress for MySQL, and
#   # partitioned reads use multiple independent JDBC statements instead of one
#   # resumable snapshot. If a migration is interrupted, it must be restarted
#   # from the beginning. For large tables, ensure your target schema uses upsert
#   # semantics (e.g. ScyllaDB INSERT) so that re-running the migration is safe.

# Example for loading from Parquet:
# source:
#   type: parquet
#   path: s3a://bucket-name/path/to/parquet-directory
#   # Optional AWS access/secret key for loading from S3.
#   # This section can be left out if running on EC2 instances that have instance profiles with the
#   # appropriate permissions.
#   credentials:
#     accessKey: <user>
#     secretKey: <pass>
#     # Optional - assume role, see https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html
#     assumeRole:
#       arn: <roleArn>
#       # Optional - the session name to use. If not set, we use 'scylla-migrator'
#       sessionName: <roleSessionName>
#   # Optional - specify the region:
#   region: <region>
#   # Optional - specify the endpoint
#   endpoint:
#     # Specify the hostname without a protocol
#     host: <host>
#     port: <port>

# Example for loading from DynamoDB:
# (Fields prefixed with ## are optional)
# source:
#   type: dynamodb
#   table: <table name>
#   ## endpoint:
#   ##   # Specify the hostname without a protocol (the AWS SDK resolves it automatically)
#   ##   host: <host>
#   ##   port: <port>
#
#   ## region: <region>
#
#   ## credentials:
#   ##   accessKey: <user>
#   ##   secretKey: <pass>
#   ##   # assume role, see https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html
#   ##   assumeRole:
#   ##     arn: <roleArn>
#   ##     sessionName: <sessionName>
#
#   # below controls split factor
#   ## scanSegments: 1
#
#   # throttling settings, set based on your capacity (or wanted capacity)
#   ## readThroughput: 1
#
#   # The value of dynamodb.throughput.read.percent can be between 0.1 and 1.5, inclusively.
#   # 0.5 represents the default read rate, meaning that the job will attempt to consume half of the read capacity of the table.
#   # If you increase the value above 0.5, spark will increase the request rate; decreasing the value below 0.5 decreases the read request rate.
#   # (The actual read rate will vary, depending on factors such as whether there is a uniform key distribution in the DynamoDB table.)
#   ## throughputReadPercent: 1.0
#
#   # how many tasks per executor?
#   ## maxMapTasks: 1

# Example for loading from a Scylla Alternator source:
# (Fields prefixed with ## are optional)
# source:
#   type: alternator
#   table: <table name>
#   # Endpoint of the Alternator instance.
#   # Unlike AWS DynamoDB, the protocol prefix ('http://' or 'https://') must be
#   # included because the endpoint is a custom URL (not resolved by the AWS SDK).
#   endpoint:
#     host: http://<alternator-host>
#     port: 8000
#
#   ## region: <region>
#
#   ## credentials:
#   ##   accessKey: <user>
#   ##   secretKey: <pass>
#
#   # below controls split factor
#   ## scanSegments: 1
#   ## readThroughput: 1
#   ## throughputReadPercent: 1.0
#   ## maxMapTasks: 1
#
#   # Remove consumed capacity headers from DynamoDB requests (defaults to true for Alternator).
#   # Needed for Scylla version > 2024.2 to avoid unnecessary capacity accounting overhead.
#   ## removeConsumedCapacity: true
#
#   # Alternator-specific settings (all optional):
#   ## datacenter: dc1
#   ## rack: rack1  (requires datacenter)
#   ## activeRefreshIntervalMs: 1000
#   ## idleRefreshIntervalMs: 60000
#   ## compression: false
#   ## optimizeHeaders: false
#   ## maxConnections: 400
#   ## connectionMaxIdleTimeMs: 600000
#   ## connectionTimeToLiveMs: 0
#   ## connectionAcquisitionTimeoutMs: 10000
#   ## connectionTimeoutMs: 15000
#   ## maxItemsPerBatch: 100

# Example for loading from a DynamoDB S3 export (see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/S3DataExport.Output.html)
# source:
#   type: dynamodb-s3-export
#   bucket: <bucket-name>
#   # Key of the `manifest-summary.json` object in the bucket
#   manifestKey: <manifest-summary-key>
#   # Key schema and attribute definitions, see https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TableCreationParameters.html
#   tableDescription:
#     attributeDefinitions:
#       - name: <attribute-name>
#         type: <attribute-type> (see https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_AttributeDefinition.html)
#       - ...
#     keySchema:
#       - name: <key-name>
#         type: <key-type> (see https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_KeySchemaElement.html)
#       - ...
#
#   # Optional - load from a custom endpoint:
#   endpoint:
#     # Specify the hostname without a protocol
#     host: <host>
#     port: <port>
#   # Optional - specify the region:
#   region: <region>
#
#   # Optional - static credentials:
#   credentials:
#     accessKey: <user>
#     secretKey: <pass>
#     # Optional - assume role, see https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html
#     assumeRole:
#       arn: <roleArn>
#       # Optional - the session name to use. If not set, we use 'scylla-migrator'
#       sessionName: <roleSessionName>
#
#   # Optional - use path-style access in S3 (see https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html)
#   usePathStyleAccess: true

# Configuration for the database you're copying into
target:
  type: scylla
  host: scylla 
  port: 9042
  #optional, if not specified None will be used
  #localDC: <localdc>
  #credentials:
  #  username: <user>
  #  password: <pass>
  # SSL as per https://github.com/scylladb/spark-cassandra-connector/blob/master/doc/reference.md#cassandra-ssl-connection-options
  #sslOptions:
  #  clientAuthEnabled: false
  #  enabled: false
  #  all below are optional! (generally just trustStorePassword and trustStorePath is needed)
  #  trustStorePassword: <pass>
  #  trustStorePath: <path>
  #  trustStoreType: JKS
  #  keyStorePassword: <pass>
  #  keyStorePath: <path>
  #  keyStoreType: JKS
  #  enabledAlgorithms:
  #   - TLS_RSA_WITH_AES_128_CBC_SHA
  #   - TLS_RSA_WITH_AES_256_CBC_SHA
  #  protocol: TLS
  # NOTE: The destination table must have the same schema as the source table.
  # If you'd like to rename columns, that's ok - see the renames parameter below.
  keyspace: stocks
  table: stocks
  # Consistency Level for the target connection
  # Options are: LOCAL_ONE, ONE, LOCAL_QUORUM, QUORUM.
  # Connector driver default is LOCAL_QUORUM.
  consistencyLevel: LOCAL_QUORUM
  # Number of connections to use to Scylla when copying
  connections: 16
  # Spark pads decimals with zeros appropriate to their scale. This causes values
  # like '3.5' to be copied as '3.5000000000...' to the target. There's no good way
  # currently to preserve the original value, so this flag can strip trailing zeros
  # on decimal values before they are written.
  stripTrailingZerosForDecimals: false
  # if we cannot persist timestamps (so preserveTimestamps==false)
  # we can enforce in writer a single TTL or writetimestamp for ALL written records
  # such writetimestamp can be e.g. set to time BEFORE starting dual writes
  # and this will make your migration safe from overwriting dual write
  # even for collections
  # ALL rows written will get the same TTL or writetimestamp or both
  # (you can uncomment just one of them, or all or none)
  # TTL in seconds (sample 7776000 is 90 days)
  #writeTTLInS: 7776000
  # writetime in microseconds (sample 1640998861000 is Saturday, January 1, 2022 2:01:01 AM GMT+01:00 )
  #writeWritetimestampInuS: 1640998861000

  # Optional - Connect via a DataStax Astra Cloud Secure Connect bundle.
  # Same path rules as the source cloud block above. See source section for details.
  #cloud:
  #  secureBundlePath: /opt/migrator/secure-connect-mycluster.zip

# Example for loading into a DataStax Astra target via a Cloud Secure Connect bundle:
# target:
#   type: scylla
#   cloud:
#     secureBundlePath: /opt/migrator/secure-connect-mycluster.zip
#   credentials:
#     username: <client-id>
#     password: <client-secret>
#   keyspace: <keyspace>
#   table: <table>
#   consistencyLevel: LOCAL_QUORUM
#   connections: 16
#   stripTrailingZerosForDecimals: false

# Example for writing into a DynamoDB target:
# (Fields prefixed with ## are optional)
# target:
#   type: dynamodb
#   table: <table name>
#   ## endpoint:
#   ##   # Specify the hostname without a protocol (the AWS SDK resolves it automatically)
#   ##   host: <host>
#   ##   port: <port>
#
#   ## region: <region>
#
#   ## credentials:
#   ##   accessKey: <user>
#   ##   secretKey: <pass>
#   ##   # assume role, see https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html
#   ##   assumeRole:
#   ##     arn: <roleArn>
#   ##     sessionName: <roleSessionName>
#
#   # throttling settings, set based on your write capacity units (or wanted capacity)
#   ## writeThroughput: 1
#
#   # The value of dynamodb.throughput.write.percent can be between 0.1 and 1.5, inclusively.
#   # 0.5 represents the default write rate, meaning that the job will attempt to consume half of the write capacity of the table.
#   # If you increase the value above 0.5, spark will increase the request rate; decreasing the value below 0.5 decreases the write request rate.
#   # (The actual write rate will vary, depending on factors such as whether there is a uniform key distribution in the DynamoDB table.)
#   ## throughputWritePercent: 1.0
#
#   # When transferring DynamoDB sources to DynamoDB targets, the migrator supports transferring live changes
#   # occuring on the source table after transferring an initial snapshot. This is done using DynamoDB streams
#   # and incurs additional charges due to the Kinesis streams created.
#   # Enable this flag to transfer live changes after transferring an initial snapshot. The migrator will continue
#   # replicating changes endlessly; it must be stopped manually.
#   #
#   # NOTE: For the migration to be performed losslessly, the initial snapshot transfer must complete within 24 hours.
#   # Otherwise, some captured changes may be lost due to the retention period of the table's stream.
#   #
#   # NOTE2: The migrator does not currently delete the created Dynamo stream. Delete it manually after ending the
#   # migrator run.
#   streamChanges: false
#
#   ## billingMode: PAY_PER_REQUEST
#
#   ## skipInitialSnapshotTransfer: false

# Example for writing into a Scylla Alternator target:
# (Fields prefixed with ## are optional)
# target:
#   type: alternator
#   table: <table name>
#   # Endpoint of the Alternator instance.
#   # Unlike AWS DynamoDB, the protocol prefix ('http://' or 'https://') must be
#   # included because the endpoint is a custom URL (not resolved by the AWS SDK).
#   endpoint:
#     host: http://<alternator-host>
#     port: 8000
#
#   ## region: <region>
#
#   ## credentials:
#   ##   accessKey: <user>
#   ##   secretKey: <pass>
#
#   # throttling settings, set based on your write capacity units (or wanted capacity)
#   ## writeThroughput: 1
#   ## throughputWritePercent: 1.0
#
#   # Stream changes from the source table after the initial snapshot transfer.
#   streamChanges: false
#
#   ## billingMode: PAY_PER_REQUEST
#
#   # Remove consumed capacity headers from DynamoDB requests (defaults to true for Alternator).
#   # Needed for Scylla version > 2024.2 to avoid unnecessary capacity accounting overhead.
#   ## removeConsumedCapacity: true
#
#   ## skipInitialSnapshotTransfer: false
#
#   # Alternator-specific settings (all optional):
#   ## datacenter: dc1
#   ## rack: rack1  (requires datacenter)
#   ## activeRefreshIntervalMs: 1000
#   ## idleRefreshIntervalMs: 60000
#   ## compression: false
#   ## optimizeHeaders: false
#   ## maxConnections: 400
#   ## connectionMaxIdleTimeMs: 600000
#   ## connectionTimeToLiveMs: 0
#   ## connectionAcquisitionTimeoutMs: 10000
#   ## connectionTimeoutMs: 15000
#   ## maxItemsPerBatch: 100

# Optional savepoint configuration for resumable migrations. Savepoints are configuration
# files (like this one), saved by the migrator as it runs. Their purpose is to skip data
# that has already been copied on a restart. This configuration is used for
# Cassandra/Scylla, DynamoDB, and Parquet sources.
#
# Omit this section for MySQL migrations. MySQL savepoints are not supported:
# Spark JDBC reads do not expose durable per-range progress, and partitioned reads
# use multiple independent JDBC statements instead of one resumable snapshot. If a
# MySQL migration is interrupted, it must be restarted from the beginning.
savepoints:
  # Where should savepoint configurations be stored? This is a path on the host running
  # the Spark driver - usually the Spark master.
  path: /app/savepoints
  #
  # The legacy path form above is equivalent to a local filesystem target:
  # target:
  #   type: filesystem
  #   path: /app/savepoints
  #
  # target.type: filesystem accepts only local filesystem paths. For Hadoop filesystem
  # URIs, use the legacy path form directly:
  # path: gs://my-bucket/scylla-migrator/savepoints
  # path: s3a://my-bucket/scylla-migrator/savepoints
  #
  # For Google Cloud Storage, the structured form below builds the Hadoop path
  # gs://my-bucket/prefix:
  # target:
  #   type: gcs
  #   bucket: my-bucket
  #   prefix: scylla-migrator/savepoints
  #   projectId: my-gcp-project
  #   # Optional GCS credentials can be supplied here for writing savepoints.
  #   # If omitted, Spark/Hadoop runtime configuration is used.
  #   # credentials:
  #   #   serviceAccountJsonKeyfile: /path/to/service-account.json
  #
  # For AWS S3, the structured form below builds the Hadoop path s3a://my-bucket/prefix:
  # target:
  #   type: s3
  #   bucket: my-bucket
  #   prefix: scylla-migrator/savepoints
  #   region: us-east-1
  #   # Optional AWS credentials can be supplied here for writing savepoints.
  #   # If the config file itself is on S3, Spark/Hadoop must still have credentials
  #   # to read that file before these settings can be parsed.
  #   # credentials:
  #   #   accessKey: ...
  #   #   secretKey: ...
  # Interval in which savepoints will be created
  intervalSeconds: 300

  # Optional - enables file-level savepoint tracking for parquet migrations.
  # When enabled (default), the migrator tracks which parquet files have been fully migrated
  # and skips them on restart, allowing for more granular resume capability.
  # Set to `false` to revert to generic behaviour without file-level savepoints.
  enableParquetFileTracking: true

# Optional - Column renaming configuration. If you'd like to rename any columns, specify them like so:
# - from: source_column_name
#   to: dest_column_name
# renames: []

# Optional - Which token ranges to skip. You shouldn't need to fill this in normally; the migrator will
# create a savepoint file with this filled.
# skipTokenRanges: []

# Optional - Which scan segments to skip. You shouldn’t need to fill this in normally; the migrator will
# create a savepoint file with this filled.
# skipSegments: []

# Configuration section for running the validator. The validator is run manually (see documentation).
# Mandatory if the application is executed in validation mode.
# validation:
#   # Should WRITETIMEs and TTLs be compared?
#   compareTimestamps: true
#   # What difference should we allow between TTLs?
#   ttlToleranceMillis: 60000
#   # What difference should we allow between WRITETIMEs?
#   writetimeToleranceMillis: 1000
#   # How many differences to fetch and print
#   failuresToFetch: 100
#   # What difference should we allow between floating point numbers?
#   floatingPointTolerance: 0.001
#   # What difference in ms should we allow between timestamps?
#   timestampMsTolerance: 0
#   # If true, rows that exist in the source but are missing in the target will be
#   # copied to the target during validation. Only missing rows are copied — rows
#   # with value differences are still only reported, never overwritten.
#   # WARNING: the validator cannot distinguish rows that were never migrated from
#   # rows deleted on the target after migration. Enabling this may resurrect rows
#   # that were intentionally deleted on the target only (e.g. during dual writes).
#   # Defaults to false.
#   copyMissingRows: false
#   # Controls the CQL TIMESTAMP used for repair writes when copyMissingRows is true
#   # and preserveTimestamps is enabled (Cassandra/Scylla only). Ignored otherwise.
#   #   source (default): use original source writetime. Preserves fidelity but the
#   #     write may be silently shadowed by a newer delete tombstone on the target.
#   #   coordinator: use current coordinator wall-clock time (µs). Beats most
#   #     tombstones but may resurrect rows deleted only on the target.
#   #   config: use the fixed value from target.writeWritetimestampInuS. That setting
#   #     must be present in the target configuration.
#   repairWritetimeStrategy: source
#   # Optional - columns to hash for MySQL-to-ScyllaDB validation (use source-side names;
#   # renames are applied automatically). This reduces Spark-side join/shuffle volume,
#   # but the validator still reads the original payload columns before hashing.
#   # hashColumns:
#   #   - large_text_column
#   #   - blob_column
#   # Optional - numeric cross-type comparison policy.
#   # Lenient (default): Int==Long==BigInt==BigDecimal(scale<=0) treated as equal;
#   #   Float/Double normalized to BigDecimal within floatingPointTolerance.
#   # DetectWiden: same as Lenient, but flags Float->Double widening that loses bits
#   #   (e.g. Float(0.1f) vs Double(0.1)) as NumericTypeMismatch.
#   # StrictType: any Float vs Double pair flagged regardless of value.
#   # numericTypePolicy: Lenient