Merge pull request #709 from datazip-inc/staging

chore: releasing version v0.3.8

Author: hash-data

destination/parquet/parquet.go

@@ -41,6 +41,7 @@ type Parquet struct {
 	basePath         string                   // construct with streamNamespace/streamName
 	partitionedFiles map[string]*FileMetadata // mapping of basePath/{regex} -> pqFiles
 	s3Client         *s3.S3
+	s3Uploader       *s3manager.Uploader
 	schema           typeutils.Fields
 }
 
@@ -77,6 +78,8 @@ func (p *Parquet) initS3Writer() error {
 		return fmt.Errorf("failed to create AWS session: %s", err)
 	}
 	p.s3Client = s3.New(sess)
+	// Initialize uploader for multipart uploads (handles files > 5GB automatically)
+	p.s3Uploader = s3manager.NewUploader(sess)
 
 	return nil
 }
@@ -282,14 +285,14 @@ func (p *Parquet) closePqFiles() error {
 			}
 			s3KeyPath = filepath.Join(s3KeyPath, parquetFile.fileName)
 
-			// Upload to S3
-			_, err = p.s3Client.PutObject(&s3.PutObjectInput{
+			// Upload to S3 using multipart upload (automatically handles files > 5GB)
+			_, err = p.s3Uploader.Upload(&s3manager.UploadInput{
 				Bucket: aws.String(p.config.Bucket),
 				Key:    aws.String(s3KeyPath),
 				Body:   file,
 			})
 			if err != nil {
-				return fmt.Errorf("failed to put object into s3: %s", err)
+				return fmt.Errorf("failed to upload object to s3: %s", err)
 			}
 
 			// Remove local file after successful upload