Snowflake Loader 0.3.0 (#1088)

Author: istreeter

docs/pipeline-components-and-applications/loaders-storage-targets/snowflake-streaming-loader/configuration-reference/_common_config.md

@@ -11,8 +11,12 @@ import Link from '@docusaurus/Link';
     <td>Optional. Default value <code>1 second</code>.  Events are emitted to Snowflake after a maximum of this duration, even if the <code>maxBytes</code> size has not been reached</td>
 </tr>
 <tr>
-    <td><code>batching.uploadConcurrency</code></td>
-    <td>Optional. Default value 3.  How many batches can we send simultaneously over the network to Snowflake</td>
+    <td><code>batching.uploadParallelismFactor</code></td>
+    <td>Optional. Default value 2.5.  Controls how many batches can we send simultaneously over the network to Snowflake. E.g. If there are 4 available processors, and <code>uploadParallelismFactor</code> is 2.5, then the loader sends up to 10 batches in parallel. Adjusting this value can cause the app to use more or less of the available CPU.</td>
+</tr>
+<tr>
+    <td><code>cpuParallelismFactor</code></td>
+    <td>Optional. Default value 0.75. Controls how the loaders splits the workload into concurrent batches which can be run in parallel. E.g. If there are 4 available processors, and <code>cpuParallelismFactor</code> is 0.75, then the loader processes 3 batches concurrently. Adjusting this value can cause the app to use more or less of the available CPU.</td>
 </tr>
 <tr>
     <td><code>retries.setupErrors.delay</code></td>
@@ -67,6 +71,10 @@ import Link from '@docusaurus/Link';
     <td><code>monitoring.webhook.tags.*</code></td>
     <td>Optional. A map of key/value strings to be included in the payload content sent to the webhook.</td>
 </tr>
+<tr>
+    <td><code>monitoring.webhook.heartbeat.*</code></td>
+    <td>Optional. Default value <code>5.minutes</code>. How often to send a heartbeat event to the webhook when healthy.</td>
+</tr>
 <tr>
     <td><code>monitoring.sentry.dsn</code></td>
     <td>Optional. Set to a Sentry URI to report unexpected runtime exceptions.</td>
@@ -95,3 +103,7 @@ import Link from '@docusaurus/Link';
     <td><code>output.good.jdbcQueryTimeout</code></td>
     <td>Optional. Sets the query timeout on the JDBC driver which connects to Snowflake</td>
 </tr>
+<tr>
+    <td><code>http.client.maxConnectionsPerServer</code></td>
+    <td> Optional. Default value 4. Configures the internal HTTP client used for alerts and telemetry. The maximum number of open HTTP requests to any single server at any one time.</td>
+</tr>

docs/pipeline-components-and-applications/loaders-storage-targets/snowflake-streaming-loader/configuration-reference/_kafka_config.md

@@ -23,6 +23,6 @@
     <td>Optional. A map of key/value pairs for <a href="https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html" target="_blank">any standard Kafka producer configuration option</a>.</td>
 </tr>
 <tr>
-    <td><code>output.bad.maxRecordSize.*</code></td>
+    <td><code>output.bad.maxRecordSize</code></td>
     <td>Optional.  Default value 1000000.  Any single failed event sent to Kafka should not exceed this size in bytes</td>
 </tr>

docs/pipeline-components-and-applications/loaders-storage-targets/snowflake-streaming-loader/configuration-reference/_kinesis_config.md

@@ -23,8 +23,16 @@
     <td>Optional. Default value 1000.  How many events the Kinesis client may fetch in a single poll.  Only used when `input.retrievalMode` is Polling.</td>
 </tr>
 <tr>
-    <td><code>input.bufferSize</code></td>
-    <td>Optional. Default value 1. The number of batches of events which are pre-fetched from kinesis. The default value is known to work well.</td>
+    <td><code>input.workerIdentifier</code></td>
+    <td>Optional. Defaults to the <code>HOSTNAME</code> environment variable. The name of this KCL worker used in the dynamodb lease table.</td>
+</tr>
+<tr>
+    <td><code>input.leaseDuration</code></td>
+    <td>Optional. Default value <code>10 seconds</code>. The duration of shard leases.  KCL workers must periodically refresh leases in the dynamodb table before this duration expires.</td>
+</tr>
+<tr>
+    <td><code>input.maxLeasesToStealAtOneTimeFactor</code></td>
+    <td>Optional. Default value <code>2.0</code>. Controls how to pick the max number of shard-leases to steal at one time. E.g. If there are 4 available processors, and <code>maxLeasesToStealAtOneTimeFactor</code> is 2.0, then allow the KCL to steal up to 8 leases. Allows bigger instances to more quickly acquire the shard-leases they need to combat latency.</td>
 </tr>
 <tr>
     <td><code>output.bad.streamName</code></td>
@@ -47,6 +55,6 @@
     <td>Optional.  Default value 5242880.  The maximum number of bytes we are allowed to send to Kinesis in 1 PutRecords request.</td>
 </tr>
 <tr>
-    <td><code>output.bad.maxRecordSize.*</code></td>
+    <td><code>output.bad.maxRecordSize</code></td>
     <td>Optional.  Default value 1000000.  Any single event failed event sent to Kinesis should not exceed this size in bytes</td>
 </tr>

docs/pipeline-components-and-applications/loaders-storage-targets/snowflake-streaming-loader/configuration-reference/_snowflake_config.md

@@ -28,9 +28,9 @@
 </tr>
 <tr>
     <td><code>output.good.table</code></td>
-    <td>Optional. Default value `events`. Name to use for the events table</td>
+    <td>Optional. Default value <code>events</code>. Name to use for the events table</td>
 </tr>
 <tr>
     <td><code>output.good.channel</code></td>
-    <td>Optional. Default value `snowplow`. Name to use for the Snowflake channel.  If you run multiple loaders in parallel, then each channel must be given a unique name.</td>
+    <td>Optional. Default value <code>snowplow</code>. Prefix to use for the snowflake channels. The full name will be suffixed with a number, e.g. <code>snowplow-1</code>. If you run multiple loaders in parallel, then each loader must be configured with a unique channel prefix.</td>
 </tr>

src/componentVersions.js

@@ -37,7 +37,7 @@ export const versions = {
   s3Loader: '2.2.9',
   s3Loader22x: '2.2.9',
   lakeLoader: '0.5.0',
-  snowflakeStreamingLoader: '0.2.4',
+  snowflakeStreamingLoader: '0.3.0',
 
   // Data Modelling
   // dbt