Merge pull request #2 from aspecto-io/fix/docs

Fix main reference, update docs

Author: mzahor

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build PR
+name: Build
 on: [push]
 
 jobs:

.npmignore

@@ -3,3 +3,6 @@
 /node_modules
 /.github
 /scripts
+.prettierrc
+docker-compose.yml
+.editorconfig

README.md

@@ -4,12 +4,25 @@
 
 SQS/SNS producer/consumer library. Provides an ability to pass payloads though s3.
 
+## Motivation
+
+[Aspecto](https://www.aspecto.io/?utm_source=github&utm_medium=sqs-sns-big-payload&utm_campaign=readme-p1&utm_content=v1) helps modern development teams solve production issues before they evolve. We collect real production data and perform deep API analysis over it to autogenerate tests and monitor services stability. As a result, we often need to handle large payloads which can't be used with SQS & SNS due to the hard size limit. This library was developed to overcome this challenge - it enables you to manage Amazon SNS & SQS message payloads with Amazon S3 when dealing with payloads larger than 256KB. Key functionality includes:
+
+-   Controlling whether message payloads are always stored in Amazon S3 or only when a message's size exceeds 256KB.
+-   Send a message that references a single message object stored in an Amazon S3 bucket.
+-   Get the corresponding message object from an Amazon S3 bucket.
+-   Handle the interface for large messages between SNS to SQS via S3 bucket in the middle
+
 ## Instalation
 
 ```
 npm install sns-sqs-big-payload
 ```
 
+Important:
+
+> Make sure you also have `aws-sdk` installed, bacause it's listed as a peer dependency, so won't be installed automatically.
+
 ## Usage
 
 The library exports 3 clients:
@@ -18,8 +31,7 @@ The library exports 3 clients:
 -   SQS producer
 -   SQS consumer
 
-The reason they belong to the same repository and npm package
-is that ther're is kind of a contract that they all share when sending the payload though S3.
+All 3 clients are under the same repository since they share a similar contract when sending payloads via S3.
 
 ### SNS Producer
 
@@ -162,10 +174,16 @@ You may subscribe to those events to add logging for example.
 
 ## Testing
 
-Since this library relies heavily on AWS API there's not much sense to test it in isolation by using mocks.
-So in order to run test you either need to have local stack or use a real sqs queues and sns topics.
+Since this library heavily relies on AWS APIs, it is less relevant to run an isolated test using mocks. As a result, we recommend testing it using a [localstack](https://github.com/localstack/localstack) or by using real SQS queues and SNS topics.
 
 To run localstack on mac:
+
 ```sh
 TMPDIR=/private$TMPDIR docker-compose up
 ```
+
+To run unit tests:
+
+```sh
+npm test
+```

package.json

@@ -1,23 +1,24 @@
 {
     "name": "sns-sqs-big-payload",
-    "version": "0.0.2",
+    "version": "0.0.3",
     "license": "MIT",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
     "scripts": {
         "test": "jest",
         "build": "npm run clean && tsc",
         "clean": "rm -rf ./dist/*"
     },
     "dependencies": {
-        "aws-sdk": "^2.644.0",
         "uuid": "^7.0.2"
     },
     "devDependencies": {
         "@types/jest": "^25.1.4",
         "@types/node": "^13.9.2",
+        "aws-sdk": "^2.644.0",
         "jest": "^25.1.0",
         "ts-jest": "^25.2.1",
-        "typescript": "^3.8.3",
-        "wait-on": "^4.0.1"
+        "typescript": "^3.8.3"
     },
     "repository": {
         "type": "git",
@@ -28,8 +29,6 @@
     },
     "homepage": "https://github.com/aspecto-io/sns-sqs-big-payload",
     "prepublish": "tsc",
-    "main": "./build/index.js",
-    "types": "./build/index.d.ts",
     "jest": {
         "preset": "ts-jest",
         "testMatch": [
@@ -58,5 +57,8 @@
         "large",
         "big",
         "payload"
-    ]
+    ],
+    "peerDependencies": {
+        "aws-sdk": "^2.644.0"
+    }
 }

src/sns-producer.ts

@@ -41,6 +41,12 @@ export class SnsProducer {
             });
         }
         if (options.allPayloadThoughS3 || options.largePayloadThoughS3) {
+            if (!options.s3Bucket) {
+                throw new Error(
+                    'Need to specify "s3Bucket" option when using allPayloadThoughS3 or  largePayloadThoughS3.'
+                );
+            }
+
             if (options.s3) {
                 this.s3 = options.s3;
             } else {

src/sqs-consumer.ts

@@ -130,7 +130,7 @@ export class SqsConsumer {
     }
 
     private async handleSqsResponse(result: ReceiveMessageResult): Promise<void> {
-        if (result) {
+        if (result && result.Messages) {
             await Promise.all(result.Messages.map((message) => this.processMessage(message)));
         }
     }

src/sqs-producer.ts

@@ -41,6 +41,11 @@ export class SqsProducer {
             });
         }
         if (options.largePayloadThoughS3 || options.allPayloadThoughS3) {
+            if (!options.s3Bucket) {
+                throw new Error(
+                    'Need to specify "s3Bucket" option when using allPayloadThoughS3 or  largePayloadThoughS3.'
+                );
+            }
             if (options.s3) {
                 this.s3 = options.s3;
             } else {

yarn.lock

@@ -151,46 +151,6 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
-"@hapi/address@^4.0.1":
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/@hapi/address/-/address-4.0.1.tgz#267301ddf7bc453718377a6fb3832a2f04a721dd"
-  integrity sha512-0oEP5UiyV4f3d6cBL8F3Z5S7iWSX39Knnl0lY8i+6gfmmIBj44JCBNtcMgwyS+5v7j3VYavNay0NFHDS+UGQcw==
-  dependencies:
-    "@hapi/hoek" "^9.0.0"
-
-"@hapi/formula@^2.0.0":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/@hapi/formula/-/formula-2.0.0.tgz#edade0619ed58c8e4f164f233cda70211e787128"
-  integrity sha512-V87P8fv7PI0LH7LiVi8Lkf3x+KCO7pQozXRssAHNXXL9L1K+uyu4XypLXwxqVDKgyQai6qj3/KteNlrqDx4W5A==
-
-"@hapi/hoek@^9.0.0":
-  version "9.0.4"
-  resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.0.4.tgz#e80ad4e8e8d2adc6c77d985f698447e8628b6010"
-  integrity sha512-EwaJS7RjoXUZ2cXXKZZxZqieGtc7RbvQhUy8FwDoMQtxWVi14tFjeFCYPZAM1mBCpOpiBpyaZbb9NeHc7eGKgw==
-
-"@hapi/joi@^17.1.0":
-  version "17.1.1"
-  resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-17.1.1.tgz#9cc8d7e2c2213d1e46708c6260184b447c661350"
-  integrity sha512-p4DKeZAoeZW4g3u7ZeRo+vCDuSDgSvtsB/NpfjXEHTUjSeINAi/RrVOWiVQ1isaoLzMvFEhe8n5065mQq1AdQg==
-  dependencies:
-    "@hapi/address" "^4.0.1"
-    "@hapi/formula" "^2.0.0"
-    "@hapi/hoek" "^9.0.0"
-    "@hapi/pinpoint" "^2.0.0"
-    "@hapi/topo" "^5.0.0"
-
-"@hapi/pinpoint@^2.0.0":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/@hapi/pinpoint/-/pinpoint-2.0.0.tgz#805b40d4dbec04fc116a73089494e00f073de8df"
-  integrity sha512-vzXR5MY7n4XeIvLpfl3HtE3coZYO4raKXW766R6DZw/6aLqR26iuZ109K7a0NtF2Db0jxqh7xz2AxkUwpUFybw==
-
-"@hapi/topo@^5.0.0":
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.0.0.tgz#c19af8577fa393a06e9c77b60995af959be721e7"
-  integrity sha512-tFJlT47db0kMqVm3H4nQYgn6Pwg10GTZHb1pwmSiv1K4ks6drQOtfEF5ZnPjkvC+y4/bUPHK+bc87QvLcL+WMw==
-  dependencies:
-    "@hapi/hoek" "^9.0.0"
-
 "@istanbuljs/load-nyc-config@^1.0.0":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.0.0.tgz#10602de5570baea82f8afbfa2630b24e7a8cfe5b"
@@ -2749,7 +2709,7 @@ request-promise-core@1.1.3:
   dependencies:
     lodash "^4.17.15"
 
-request-promise-native@^1.0.7, request-promise-native@^1.0.8:
+request-promise-native@^1.0.7:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/request-promise-native/-/request-promise-native-1.0.8.tgz#a455b960b826e44e2bf8999af64dff2bfe58cb36"
   integrity sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ==
@@ -2840,13 +2800,6 @@ rsvp@^4.8.4:
   resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734"
   integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==
 
-rxjs@^6.5.4:
-  version "6.5.4"
-  resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.4.tgz#e0777fe0d184cec7872df147f303572d414e211c"
-  integrity sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==
-  dependencies:
-    tslib "^1.9.0"
-
 safe-buffer@^5.0.1, safe-buffer@^5.1.2:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519"
@@ -3282,11 +3235,6 @@ ts-jest@^25.2.1:
     semver "^5.5"
     yargs-parser "^16.1.0"
 
-tslib@^1.9.0:
-  version "1.11.1"
-  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.11.1.tgz#eb15d128827fbee2841549e171f45ed338ac7e35"
-  integrity sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==
-
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
@@ -3430,18 +3378,6 @@ w3c-xmlserializer@^1.1.2:
     webidl-conversions "^4.0.2"
     xml-name-validator "^3.0.0"
 
-wait-on@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/wait-on/-/wait-on-4.0.1.tgz#c49ca18b1ea60580404feed9df76ab3af2425a56"
-  integrity sha512-x83fmTH2X0KL7vXoGt9aV5x4SMCvO8A/NbwWpaYYh4NJ16d3KSgbHwBy9dVdHj0B30cEhOFRvDob4fnpUmZxvA==
-  dependencies:
-    "@hapi/joi" "^17.1.0"
-    lodash "^4.17.15"
-    minimist "^1.2.0"
-    request "^2.88.0"
-    request-promise-native "^1.0.8"
-    rxjs "^6.5.4"
-
 walker@^1.0.7, walker@~1.0.5:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.7.tgz#2f7f9b8fd10d677262b18a884e28d19618e028fb"