Add support for visibility timeout in AzureQueueStore

Implement visibility timeout functionality in AzureQueueStore to allow messages to be hidden for a specified duration after being pushed onto the queue. This enhancement works in conjunction with the improvements in definition computation on the service side, reducing the number of definition computations when component harvest results are available.

Author: qtomlinson

config/cdConfig.js

@@ -117,7 +117,8 @@ module.exports = {
       account: cd_azblob.account,
       queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests',
       spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'),
-      isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false
+      isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false,
+      visibilityTimeout: parseInt(config.get('CRAWLER_HARVESTS_QUEUE_VISIBILITY_TIMEOUT_SECONDS')) || 0
     },
     'cd(azblob)': cd_azblob,
     'cd(file)': cd_file

providers/store/azureQueueStore.js

@@ -54,7 +54,8 @@ class AzureStorageQueue {
 
   async upsert(document) {
     const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64')
-    return await this.queueService.sendMessage(message)
+    const options = { visibilityTimeout: this.options.visibilityTimeout || 0 }
+    return await this.queueService.sendMessage(message, options)
   }
 
   get() {