Skip to content

Commit 9001d3e

Browse files
rajadilipkollirajadilipkolli
authored
feat : expose endpoints for metrics (#916)
* feat : expose endpoints for metrics * polish * fix tests * move calculting to db * handle test assertion * fix * add composite index on bucket_name and file_size in init-db.sql * refactor testGetContentTypeDistribution for conciseness using Collectors.toMap
1 parent d363201 commit 9001d3e

11 files changed

Lines changed: 497 additions & 30 deletions

File tree

‎aws-s3-project/ReadMe.md‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,65 @@
1-
# aws-s3-project (using amazon V2)
1+
# aws-s3-project (using AWS SDK V2)
22

3-
AWS S3 (Amazon Web Services Simple Storage Service) is a cloud storage service offered by Amazon Web Services (AWS). It allows users to store and retrieve large amounts of data, such as files, images, and videos, in a highly scalable and secure manner. S3 provides various features, such as access control, data encryption, and automatic data replication across multiple data centers, to ensure data durability and availability. It is commonly used by businesses and organizations for data backup, disaster recovery, and data storage for applications and websites.
3+
AWS S3 (Amazon Web Services Simple Storage Service) is a cloud storage service that allows users to store and retrieve large amounts of data in a highly scalable and secure manner. This project demonstrates integration with AWS S3 using Spring Cloud AWS and AWS SDK V2.
44

5-
### Run tests
6-
`$ ./mvnw clean verify`
5+
## Features
76

8-
### Run locally
7+
- **File Upload/Download**: Upload files to S3 and download them directly
8+
- **Pre-Signed URLs**: Generate pre-signed URLs for secure temporary access to S3 objects
9+
- **S3 Bucket Management**: Create buckets and list objects
10+
- **File Metadata**: Track and manage file metadata in a database
11+
- **Server-Side Encryption**: Secure objects with server-side encryption
12+
- **Object Versioning**: Track multiple versions of objects
13+
- **Object Tagging**: Add custom tags to S3 objects for better organization
14+
- **Storage Metrics**: Track and analyze storage usage patterns
15+
16+
## API Endpoints
17+
18+
### File Operations
19+
- `POST /s3/upload` - Upload a file directly to S3
20+
- `POST /s3/upload/signed/` - Upload a file using a pre-signed URL
21+
- `GET /s3/download/{name}` - Download a file directly from S3
22+
- `GET /s3/download/signed/{bucketName}/{name}` - Get a pre-signed URL to download a file
23+
- `GET /s3/view-all` - List all objects in the default S3 bucket
24+
- `GET /s3/view-all-db` - List all file metadata from the database
25+
26+
### Object Tagging
27+
- `POST /s3/tags` - Add or update tags for an object
28+
- `GET /s3/tags/{fileName}` - Get all tags for an object
29+
30+
### Storage Metrics
31+
- `GET /s3/metrics` - Get overall storage metrics
32+
- `GET /s3/metrics/bucket/{bucketName}` - Get metrics for a specific bucket
33+
34+
## Configuration
35+
36+
The application can be configured using the following properties in `application.yml`:
37+
38+
```yaml
39+
application:
40+
bucket-name: your-bucket-name
41+
enable-server-side-encryption: true # Enable/disable server-side encryption
42+
server-side-encryption-algorithm: AES256 # Encryption algorithm to use
43+
enable-versioning: true # Enable/disable object versioning
44+
```
45+
46+
## Run Tests
47+
```
48+
$ ./mvnw clean verify
49+
```
50+
51+
## Run Locally
952
```
1053
$ docker-compose -f docker/docker-compose.yml up -d
1154
$ ./mvnw spring-boot:run -Dspring-boot.run.profiles=local
1255
```
1356

14-
### Using Testcontainers at Development Time
57+
## Using Testcontainers at Development Time
1558
```shell
1659
./mvnw spotless:apply spring-boot:test-run
1760
```
1861

19-
### Useful Links
62+
## Useful Links
2063
* Swagger UI: http://localhost:8080/swagger-ui.html
2164
* Actuator Endpoint: http://localhost:8080/actuator
2265
* Prometheus: http://localhost:9090/

‎aws-s3-project/docker/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version: '3.8'
1+
name: aws-s3-project-localstack
22
services:
33

44
postgresqldb:

‎aws-s3-project/src/main/java/com/learning/awspring/config/S3Config.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ private void configureVersioning(String bucketName) {
6262
.build());
6363

6464
log.info(
65-
"Versioning enabling status for bucket: {} is {}",
65+
"Versioning status for bucket: {} is {}",
6666
bucketName,
6767
putBucketVersioningResponse.sdkHttpResponse().isSuccessful());
6868
} else {

‎aws-s3-project/src/main/java/com/learning/awspring/controller/FileInfoController.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ ResponseEntity<InputStreamResource> downloadFromS3Route(
7777
SignedURLResponse downloadFileUsingSignedURL(
7878
@PathVariable String bucketName,
7979
@PathVariable("name") String fileName,
80-
@RequestParam(value = "durationSeconds", required = false) Integer durationSeconds) {
80+
@RequestParam(value = "durationSeconds", required = false) Long durationSeconds) {
8181
if (durationSeconds != null) {
8282
return awsS3Service.downloadFileUsingSignedURL(
8383
bucketName, fileName, Duration.ofSeconds(durationSeconds));

‎aws-s3-project/src/main/java/com/learning/awspring/controller/StorageMetricsController.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.learning.awspring.controller;
2+
3+
import com.learning.awspring.service.StorageMetricsService;
4+
import java.util.Map;
5+
import org.springframework.http.ResponseEntity;
6+
import org.springframework.web.bind.annotation.GetMapping;
7+
import org.springframework.web.bind.annotation.PathVariable;
8+
import org.springframework.web.bind.annotation.RequestMapping;
9+
import org.springframework.web.bind.annotation.RestController;
10+
11+
@RestController
12+
@RequestMapping("/s3/metrics")
13+
public class StorageMetricsController {
14+
15+
private final StorageMetricsService storageMetricsService;
16+
17+
public StorageMetricsController(StorageMetricsService storageMetricsService) {
18+
this.storageMetricsService = storageMetricsService;
19+
}
20+
21+
/**
22+
* Get overall storage metrics for all buckets.
23+
*
24+
* @return A map of storage metrics
25+
*/
26+
@GetMapping
27+
public ResponseEntity<Map<String, Object>> getStorageMetrics() {
28+
return ResponseEntity.ok(storageMetricsService.getStorageMetrics());
29+
}
30+
31+
/**
32+
* Get storage metrics for a specific bucket.
33+
*
34+
* @param bucketName The name of the bucket to get metrics for
35+
* @return A map of storage metrics for the bucket
36+
*/
37+
@GetMapping("/bucket/{bucketName}")
38+
public ResponseEntity<Map<String, Object>> getBucketMetrics(@PathVariable String bucketName) {
39+
return ResponseEntity.ok(storageMetricsService.getBucketMetrics(bucketName));
40+
}
41+
}

‎aws-s3-project/src/main/java/com/learning/awspring/repository/FileInfoRepository.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,48 @@
11
package com.learning.awspring.repository;
22

33
import com.learning.awspring.entities.FileInfo;
4+
import java.time.LocalDateTime;
45
import java.util.List;
56
import org.springframework.data.jpa.repository.JpaRepository;
7+
import org.springframework.data.jpa.repository.Query;
8+
import org.springframework.data.repository.query.Param;
69

710
public interface FileInfoRepository extends JpaRepository<FileInfo, Integer> {
811
List<FileInfo> findByFileName(String name);
912

13+
List<FileInfo> findByBucketName(String bucketName);
14+
1015
boolean existsByFileName(String fileName);
16+
17+
List<FileInfo> findByContentType(String contentType);
18+
19+
List<FileInfo> findByCreatedAtAfter(LocalDateTime timestamp);
20+
21+
@Query("SELECT f FROM FileInfo f WHERE f.fileSize > :minSize")
22+
List<FileInfo> findLargeFiles(@Param("minSize") Long minSizeBytes);
23+
24+
@Query("SELECT sum(f.fileSize) FROM FileInfo f WHERE f.bucketName = :bucketName")
25+
Long getTotalSizeByBucket(@Param("bucketName") String bucketName);
26+
27+
@Query("SELECT COUNT(f) FROM FileInfo f")
28+
Long getTotalFileCount();
29+
30+
@Query("SELECT SUM(f.fileSize) FROM FileInfo f")
31+
Long getTotalStorageBytes();
32+
33+
@Query("SELECT COUNT(f) FROM FileInfo f WHERE f.createdAt > :since")
34+
Long getRecentFileCount(@Param("since") LocalDateTime since);
35+
36+
@Query("SELECT f.contentType, COUNT(f) FROM FileInfo f GROUP BY f.contentType")
37+
List<Object[]> getContentTypeDistribution();
38+
39+
@Query("SELECT f.bucketName, COUNT(f) FROM FileInfo f GROUP BY f.bucketName")
40+
List<Object[]> getBucketDistribution();
41+
42+
@Query("SELECT COUNT(f) FROM FileInfo f WHERE f.bucketName = :bucketName")
43+
Long getFileCountByBucket(@Param("bucketName") String bucketName);
44+
45+
@Query(
46+
"SELECT f FROM FileInfo f WHERE f.bucketName = :bucketName AND f.fileSize = (SELECT MAX(ff.fileSize) FROM FileInfo ff WHERE ff.bucketName = :bucketName)")
47+
List<FileInfo> findLargestFileInBucket(@Param("bucketName") String bucketName);
1148
}

‎aws-s3-project/src/main/java/com/learning/awspring/service/StorageMetricsService.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package com.learning.awspring.service;
2+
3+
import com.learning.awspring.entities.FileInfo;
4+
import com.learning.awspring.repository.FileInfoRepository;
5+
import java.time.LocalDateTime;
6+
import java.util.*;
7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
9+
import org.springframework.stereotype.Service;
10+
import org.springframework.transaction.annotation.Transactional;
11+
12+
@Service
13+
@Transactional(readOnly = true)
14+
public class StorageMetricsService {
15+
16+
private static final Logger log = LoggerFactory.getLogger(StorageMetricsService.class);
17+
private static final double BYTES_TO_MB = 1024.0 * 1024.0;
18+
19+
private final FileInfoRepository fileInfoRepository;
20+
21+
public StorageMetricsService(FileInfoRepository fileInfoRepository) {
22+
this.fileInfoRepository = fileInfoRepository;
23+
}
24+
25+
/**
26+
* Calculate and return storage metrics for the files stored in S3.
27+
*
28+
* @return A map containing storage metrics
29+
*/
30+
public Map<String, Object> getStorageMetrics() {
31+
log.debug("Calculating overall storage metrics using database aggregation");
32+
33+
// Check if any files exist in the repository
34+
Long totalFileCount = fileInfoRepository.getTotalFileCount();
35+
if (totalFileCount == null || totalFileCount == 0) {
36+
log.debug("No files found, returning empty metrics");
37+
return Collections.emptyMap();
38+
}
39+
40+
Map<String, Object> metrics = new HashMap<>();
41+
42+
// Total file count - using direct query
43+
metrics.put("totalFileCount", totalFileCount);
44+
45+
// Total storage used (in bytes) - using direct query
46+
Long totalStorageBytes = fileInfoRepository.getTotalStorageBytes();
47+
totalStorageBytes = totalStorageBytes != null ? totalStorageBytes : 0L;
48+
metrics.put("totalStorageBytes", totalStorageBytes);
49+
metrics.put(
50+
"totalStorageMB", totalStorageBytes > 0 ? totalStorageBytes / BYTES_TO_MB : 0.0);
51+
52+
// Files uploaded in the last 24 hours - using direct query
53+
LocalDateTime oneDayAgo = LocalDateTime.now().minusDays(1);
54+
Long recentFileCount = fileInfoRepository.getRecentFileCount(oneDayAgo);
55+
metrics.put("filesUploadedLast24Hours", recentFileCount != null ? recentFileCount : 0);
56+
57+
// Content type distribution - using direct query
58+
Map<String, Long> contentTypeDistribution = new HashMap<>();
59+
List<Object[]> contentTypeResults = fileInfoRepository.getContentTypeDistribution();
60+
for (Object[] result : contentTypeResults) {
61+
if (result[0] != null) {
62+
contentTypeDistribution.put((String) result[0], (Long) result[1]);
63+
}
64+
}
65+
metrics.put("contentTypeDistribution", contentTypeDistribution);
66+
67+
// Bucket distribution - using direct query
68+
Map<String, Long> bucketDistribution = new HashMap<>();
69+
List<Object[]> bucketResults = fileInfoRepository.getBucketDistribution();
70+
for (Object[] result : bucketResults) {
71+
if (result[0] != null) {
72+
bucketDistribution.put((String) result[0], (Long) result[1]);
73+
}
74+
}
75+
metrics.put("bucketDistribution", bucketDistribution);
76+
77+
log.debug("Calculated metrics using database aggregation");
78+
return metrics;
79+
}
80+
81+
/**
82+
* Get file storage metrics for a specific bucket.
83+
*
84+
* @param bucketName The name of the bucket
85+
* @return A map containing storage metrics for the bucket
86+
*/
87+
public Map<String, Object> getBucketMetrics(String bucketName) {
88+
log.debug("Calculating bucket metrics for {} using database aggregation", bucketName);
89+
Map<String, Object> metrics = new HashMap<>();
90+
91+
metrics.put("bucketName", bucketName);
92+
93+
// Get file count using direct query
94+
Long fileCount = fileInfoRepository.getFileCountByBucket(bucketName);
95+
metrics.put("fileCount", fileCount != null ? fileCount : 0);
96+
97+
// Get total storage size using direct query
98+
Long totalBucketStorageBytes = fileInfoRepository.getTotalSizeByBucket(bucketName);
99+
totalBucketStorageBytes = totalBucketStorageBytes != null ? totalBucketStorageBytes : 0L;
100+
metrics.put("totalStorageBytes", totalBucketStorageBytes);
101+
metrics.put(
102+
"totalStorageMB",
103+
totalBucketStorageBytes > 0 ? totalBucketStorageBytes / BYTES_TO_MB : 0.0);
104+
105+
// Get the largest file in bucket using direct query
106+
List<FileInfo> largestFiles = fileInfoRepository.findLargestFileInBucket(bucketName);
107+
if (!largestFiles.isEmpty()) {
108+
FileInfo fileInfo = largestFiles.getFirst();
109+
Map<String, Object> largestFileInfo = new HashMap<>();
110+
largestFileInfo.put("fileName", fileInfo.getFileName());
111+
largestFileInfo.put("fileSize", fileInfo.getFileSize());
112+
largestFileInfo.put("contentType", fileInfo.getContentType());
113+
metrics.put("largestFile", largestFileInfo);
114+
}
115+
116+
log.debug("Calculated bucket metrics for {} using database aggregation", bucketName);
117+
return metrics;
118+
}
119+
}

‎aws-s3-project/src/main/resources/db/changelog/init-db.sql‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,7 @@ CREATE TABLE IF NOT EXISTS file_info (
1313
bucket_name varchar(64) NOT NULL,
1414
created_at timestamp DEFAULT CURRENT_TIMESTAMP,
1515
updated_at timestamp DEFAULT CURRENT_TIMESTAMP
16-
);
16+
);
17+
18+
-- Create composite index on bucket_name and file_size
19+
CREATE INDEX idx_bucket_file_size ON file_info(bucket_name, file_size);

‎aws-s3-project/src/test/java/com/learning/awspring/common/LocalStackContainerConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ public class LocalStackContainerConfig {
1414
@Bean
1515
LocalStackContainer localstackContainer() {
1616
return new LocalStackContainer(
17-
DockerImageName.parse("localstack/localstack").withTag("4.4.0"))
17+
DockerImageName.parse("localstack/localstack").withTag("4.5.0"))
1818
.withCopyFileToContainer(
1919
MountableFile.forHostPath(".localstack/"), "/etc/localstack/init/ready.d/")
2020
.waitingFor(Wait.forLogMessage(".*LocalStack initialized successfully\n", 1));

0 commit comments

Comments
 (0)