StaticSite: support removing old files (#1541)

* StaticSite: support removing old files

* Sync

Author: fwang

packages/resources/assets/BaseSite/archiver.js

@@ -107,5 +107,9 @@ function generateZips() {
     }
 
     await archive.finalize();
+
+    // Create a filenames file
+    const filenamesPath = path.join(ZIP_PATH, `filenames`);
+    fs.writeFileSync(filenamesPath, files.join("\n"));
   });
 }

packages/resources/assets/BaseSite/custom-resource/s3-handler.py

@@ -50,6 +50,7 @@ def cfn_error(message=None):
         try:
             sources            = props['Sources']
             dest_bucket_name   = props['DestinationBucketName']
+            filenames          = props.get('Filenames', None)
             file_options       = props.get('FileOptions', [])
             replace_values     = props.get('ReplaceValues', [])
         except KeyError as e:
@@ -69,6 +70,9 @@ def cfn_error(message=None):
         if request_type == "Update" or request_type == "Create":
             loop = asyncio.get_event_loop()
             loop.run_until_complete(s3_deploy_all(sources, dest_bucket_name, file_options, replace_values))
+            # purge old items
+            if filenames:
+                s3_purge(filenames, dest_bucket_name)
 
         cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id)
     except KeyError as e:
@@ -124,6 +128,32 @@ def s3_deploy(function_name, source, dest_bucket_name, file_options, replace_val
     if (result['Status'] != True):
         raise Exception("failed to upload to s3")
 
+#---------------------------------------------------------------------------------------------------
+# remove old files
+def s3_purge(filenames, dest_bucket_name):
+    logger.info("| s3_purge")
+
+    source_bucket_name = filenames['BucketName']
+    source_object_key  = filenames['ObjectKey']
+    s3_source = "s3://%s/%s" % (source_bucket_name, source_object_key)
+
+    # create a temporary working directory
+    workdir=tempfile.mkdtemp()
+    logger.info("| workdir: %s" % workdir)
+
+    # download the archive from the source and extract to "contents"
+    target_path=os.path.join(workdir, str(uuid4()))
+    logger.info("target_path: %s" % target_path)
+    aws_command("s3", "cp", s3_source, target_path)
+    with open(target_path) as f:
+        filepaths = f.read().splitlines()
+
+    #s3_dest get S3 files
+    for file in s3.Bucket(dest_bucket_name).objects.all():
+        if (file.key not in filepaths):
+            logger.info("| removing file %s", file.key)
+            file.delete()
+
 #---------------------------------------------------------------------------------------------------
 # executes an "aws" cli command
 def aws_command(*args):

packages/resources/src/StaticSite.ts

@@ -46,6 +46,7 @@ export interface StaticSiteProps {
   readonly s3Bucket?: s3.BucketProps;
   readonly cfDistribution?: StaticSiteCdkDistributionProps;
   readonly environment?: { [key: string]: string };
+  readonly purgeFiles?: boolean;
   readonly disablePlaceholder?: boolean;
   readonly waitForInvalidation?: boolean;
 }
@@ -68,6 +69,7 @@ export class StaticSite extends Construct implements SSTConstruct {
   private readonly props: StaticSiteProps;
   private readonly isPlaceholder: boolean;
   private readonly assets: s3Assets.Asset[];
+  private readonly filenamesAsset?: s3Assets.Asset;
   private readonly awsCliLayer: AwsCliLayer;
 
   constructor(scope: Construct, id: string, props: StaticSiteProps) {
@@ -92,7 +94,9 @@ export class StaticSite extends Construct implements SSTConstruct {
     this.validateCustomDomainSettings();
 
     // Build app
-    this.assets = this.buildApp(fileSizeLimit, buildDir);
+    this.buildApp();
+    this.assets = this.bundleAssets(fileSizeLimit, buildDir);
+    this.filenamesAsset = this.bundleFilenamesAsset(buildDir);
 
     // Create Bucket
     this.s3Bucket = this.createS3Bucket();
@@ -159,17 +163,12 @@ export class StaticSite extends Construct implements SSTConstruct {
     };
   }
 
-  private buildApp(fileSizeLimit: number, buildDir: string): s3Assets.Asset[] {
+  private buildApp() {
     if (this.isPlaceholder) {
-      return [
-        new s3Assets.Asset(this, "Asset", {
-          path: path.resolve(__dirname, "../assets/StaticSite/stub"),
-        }),
-      ];
+      return;
     }
 
     const { path: sitePath, buildCommand } = this.props;
-    const buildOutput = this.props.buildOutput || ".";
 
     // validate site path exists
     if (!fs.existsSync(sitePath)) {
@@ -180,8 +179,6 @@ export class StaticSite extends Construct implements SSTConstruct {
       );
     }
 
-    // Build and package user's website
-
     // build
     if (buildCommand) {
       try {
@@ -200,6 +197,22 @@ export class StaticSite extends Construct implements SSTConstruct {
         );
       }
     }
+  }
+
+  private bundleAssets(
+    fileSizeLimit: number,
+    buildDir: string
+  ): s3Assets.Asset[] {
+    if (this.isPlaceholder) {
+      return [
+        new s3Assets.Asset(this, "Asset", {
+          path: path.resolve(__dirname, "../assets/StaticSite/stub"),
+        }),
+      ];
+    }
+
+    const { path: sitePath, buildCommand } = this.props;
+    const buildOutput = this.props.buildOutput || ".";
 
     // validate buildOutput exists
     const siteOutputPath = path.resolve(path.join(sitePath, buildOutput));
@@ -248,6 +261,31 @@ export class StaticSite extends Construct implements SSTConstruct {
     return assets;
   }
 
+  private bundleFilenamesAsset(buildDir: string): s3Assets.Asset | undefined {
+    if (this.isPlaceholder) {
+      return;
+    }
+    if (this.props.purgeFiles === false) {
+      return;
+    }
+
+    const zipPath = path.resolve(
+      path.join(buildDir, `StaticSite-${this.node.id}-${this.node.addr}`)
+    );
+
+    // create assets
+    const filenamesPath = path.join(zipPath, `filenames`);
+    if (!fs.existsSync(filenamesPath)) {
+      throw new Error(
+        `There was a problem generating the "${this.node.id}" StaticSite package.`
+      );
+    }
+
+    return new s3Assets.Asset(this, `AssetFilenames`, {
+      path: filenamesPath,
+    });
+  }
+
   private createS3Bucket(): s3.Bucket {
     let { s3Bucket } = this.props;
     s3Bucket = s3Bucket || {};
@@ -304,6 +342,7 @@ export class StaticSite extends Construct implements SSTConstruct {
       },
     });
     this.s3Bucket.grantReadWrite(handler);
+    this.filenamesAsset?.grantRead(handler);
     uploader.grantInvoke(handler);
 
     // Create custom resource
@@ -316,6 +355,10 @@ export class StaticSite extends Construct implements SSTConstruct {
           ObjectKey: asset.s3ObjectKey,
         })),
         DestinationBucketName: this.s3Bucket.bucketName,
+        Filenames: this.filenamesAsset && {
+          BucketName: this.filenamesAsset.s3BucketName,
+          ObjectKey: this.filenamesAsset.s3ObjectKey,
+        },
         FileOptions: (fileOptions || []).map(
           ({ exclude, include, cacheControl }) => {
             if (typeof exclude === "string") {

www/docs/constructs/StaticSite.md

@@ -639,6 +639,12 @@ _Type_: [`StaticSiteCdkDistributionProps`](#staticsitecdkdistributionprops)
 
 Pass in a `StaticSiteCdkDistributionProps` value to override the default settings this construct uses to create the CDK `Distribution` internally.
 
+### purgeFiles?
+
+_Type_ : `boolean`, _defaults to true_
+
+While deploying, SST removes old files that no longer exist. Pass in `false` to keep the old files around.
+
 ### waitForInvalidation?
 
 _Type_ : `boolean`, _defaults to true_