[IT-4485] Upload CSV files from MIP to FloQast

.coveragerc

@@ -2,4 +2,4 @@
 relative_files = True
 
 # Use 'source' instead of 'omit' in order to ignore 'tests/unit/__init__.py'
-source = hello_world
+source = floqast_sftp

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ ci:
   autoupdate_schedule: monthly
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
     # On Windows, git will convert all CRLF to LF, but only after all hooks are done executing.
     # yamllint will fail before git has a chance to convert line endings, so line endings must be explicitly converted before yamllint
@@ -17,7 +17,7 @@ repos:
     hooks:
     -   id: yamllint
 -   repo: https://github.com/awslabs/cfn-python-lint
-    rev: v1.35.3
+    rev: v1.38.2
     hooks:
     -   id: cfn-python-lint
         files: template\.(json|yml|yaml)$
@@ -29,3 +29,8 @@ repos:
     rev: v1.2.0
     hooks:
       - id: j2lint
+-   repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+    - id: black
+      language_version: python3.12

Pipfile

@@ -6,11 +6,15 @@ verify_ssl = true
 [dev-packages]
 pytest = "~=7.1"
 pytest-mock = "~=3.8"
+requests-mock = "~=1.10"
+paramiko-mock = "~=1.0"
 boto3 = "~=1.24"
 coverage = "~=7.3"
 
 [packages]
 crhelper = "~=2.0"
+paramiko = "~=3.5"
+requests = "~=2.32"
 
 [requires]
 python_version = "3.12"

README.md

@@ -1,11 +1,49 @@
-# lambda-template
-A GitHub template for quickly starting a new AWS lambda project.
+# lambda-finops-floqast-sftp
+
+AWS Lambda to periodically fetch trial balances from lambda-mips-api as CSV files
+and upload them to FloQast via SFTP.
+
+The activity period for the trial balances is one month, or month-to-date for the
+current month. A configurable number of activity periods will be fetched, starting
+with the current month and moving backwards. A separate CSV file will be fetched
+for each activity period, and uploaded to the specified SFTP server with a unique
+file name.
+
+## Parameters
+
+### SSM Parameters
+
+User credentials for logging in to the SFTP server are stored as secure
+parameters in SSM with a configurable prefix. By default, the prefix is `/floqast-sftp`.
+
+#### Required SSM Parameters
+
+The `user`, `pass`, and `host` parameters are required for SFTP authentication.
+
+| Parameter | Description    |
+|-----------|----------------|
+| user      | SFTP username  |
+| pass      | SFTP password  |
+| host      | SFTP host name |
+
+#### Optional SSM Parameter
+
+An optional `port` parameter can be used to configure the host port.
+
+| Parameter | Description    | Default |
+|-----------|----------------|--------|
+| port      | SFTP host port | 22     |
+
+### Template Parameters
+
+The following template parameters are used to configure behavior:
+
+| Template Parameter | Type                            | Default               | Description                                                                                        |
+|---------------------|---------------------------------|-----------------------|----------------------------------------------------------------------------------------------------|
+| Schedule            | EventBridge Schedule Expression | `cron(30 10 2 * ? *)` | EventBridge schedule for running the lambda                                                        |
+| SsmPrefix           | String                          | /floqast-sftp         | Prepend this value to the SSM parameter keys                                                       |
+| PeriodCount         | Number                          | 2                     | The number of activity periods (months) to report on, starting at the present and moving backwards |
 
-## Naming
-Naming conventions:
-* for a vanilla Lambda: `lambda-<context>`
-* for a Cloudformation Transform macro: `cfn-macro-<context>`
-* for a Cloudformation Custom Resource: `cfn-cr-<context>`
 
 ## Development
 
@@ -75,7 +113,7 @@ Running integration tests
 [requires docker](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-local-start-api.html)
 
 ```shell script
-$ sam local invoke HelloWorldFunction --event events/event.json
+$ sam local invoke FloQastSftpFunction --event events/event.json
 ```
 
 ## Deployment
@@ -90,9 +128,9 @@ which requires permissions to upload to Sage
 ```shell script
 sam package --template-file .aws-sam/build/template.yaml \
   --s3-bucket essentials-awss3lambdaartifactsbucket-x29ftznj6pqw \
-  --output-template-file .aws-sam/build/lambda-template.yaml
+  --output-template-file .aws-sam/build/lambda-finops-floqast-sftp.yaml
 
-aws s3 cp .aws-sam/build/lambda-template.yaml s3://bootstrap-awss3cloudformationbucket-19qromfd235z9/lambda-template/main/
+aws s3 cp .aws-sam/build/lambda-finops-floqast-sftp.yaml s3://bootstrap-awss3cloudformationbucket-19qromfd235z9/lambda-finops-floqast-sftp/main/
 ```
 
 ## Publish Lambda
@@ -102,7 +140,7 @@ Publishing the lambda makes it available in your AWS account.  It will be access
 the [serverless application repository](https://console.aws.amazon.com/serverlessrepo).
 
 ```shell script
-sam publish --template .aws-sam/build/lambda-template.yaml
+sam publish --template .aws-sam/build/lambda-finops-floqast-sftp.yaml
 ```
 
 ### Public access
@@ -119,13 +157,13 @@ aws serverlessrepo put-application-policy \
 
 ### Sceptre
 Create the following [sceptre](https://github.com/Sceptre/sceptre) file
-config/prod/lambda-template.yaml
+config/prod/lambda-finops-floqast-sftp.yaml
 
 ```yaml
 template:
   type: http
-  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-template/VERSION/lambda-template.yaml"
-stack_name: "lambda-template"
+  url: "https://PUBLISH_BUCKET.s3.amazonaws.com/lambda-finops-floqast-sftp/VERSION/lambda-finops-floqast-sftp.yaml"
+stack_name: "lambda-finops-floqast-sftp"
 stack_tags:
   Department: "Platform"
   Project: "Infrastructure"
@@ -134,7 +172,7 @@ stack_tags:
 
 Install the lambda using sceptre:
 ```shell script
-sceptre --var "profile=my-profile" --var "region=us-east-1" launch prod/lambda-template.yaml
+sceptre --var "profile=my-profile" --var "region=us-east-1" launch prod/lambda-finops-floqast-sftp.yaml
 ```
 
 ### AWS Console

events/event.json

@@ -1,62 +1,4 @@
 {
-  "body": "{\"message\": \"hello world\"}",
-  "resource": "/{proxy+}",
-  "path": "/path/to/resource",
-  "httpMethod": "POST",
-  "isBase64Encoded": false,
-  "queryStringParameters": {
-    "foo": "bar"
-  },
-  "pathParameters": {
-    "proxy": "/path/to/resource"
-  },
-  "stageVariables": {
-    "baz": "qux"
-  },
-  "headers": {
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "Accept-Encoding": "gzip, deflate, sdch",
-    "Accept-Language": "en-US,en;q=0.8",
-    "Cache-Control": "max-age=0",
-    "CloudFront-Forwarded-Proto": "https",
-    "CloudFront-Is-Desktop-Viewer": "true",
-    "CloudFront-Is-Mobile-Viewer": "false",
-    "CloudFront-Is-SmartTV-Viewer": "false",
-    "CloudFront-Is-Tablet-Viewer": "false",
-    "CloudFront-Viewer-Country": "US",
-    "Host": "1234567890.execute-api.us-east-1.amazonaws.com",
-    "Upgrade-Insecure-Requests": "1",
-    "User-Agent": "Custom User Agent String",
-    "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)",
-    "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==",
-    "X-Forwarded-For": "127.0.0.1, 127.0.0.2",
-    "X-Forwarded-Port": "443",
-    "X-Forwarded-Proto": "https"
-  },
-  "requestContext": {
-    "accountId": "123456789012",
-    "resourceId": "123456",
-    "stage": "prod",
-    "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef",
-    "requestTime": "09/Apr/2015:12:34:56 +0000",
-    "requestTimeEpoch": 1428582896000,
-    "identity": {
-      "cognitoIdentityPoolId": null,
-      "accountId": null,
-      "cognitoIdentityId": null,
-      "caller": null,
-      "accessKey": null,
-      "sourceIp": "127.0.0.1",
-      "cognitoAuthenticationType": null,
-      "cognitoAuthenticationProvider": null,
-      "userArn": null,
-      "userAgent": "Custom User Agent String",
-      "user": null
-    },
-    "path": "/prod/path/to/resource",
-    "resourcePath": "/{proxy+}",
-    "httpMethod": "POST",
-    "apiId": "1234567890",
-    "protocol": "HTTP/1.1"
-  }
+    "ssm_secret_prefix": "test/",
+    "period_count": "2"
 }