Skip to content

chore: [PAGOPA-2289] Massive queries #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Jun 20, 2025
Merged

chore: [PAGOPA-2289] Massive queries #10

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
057c116
improvements
aomegax Jun 5, 2025
cd4d58f
fix unit test
aomegax Jun 5, 2025
b432e6c
fix sonar
aomegax Jun 6, 2025
2dbd997
fix sonar
aomegax Jun 6, 2025
9e990a2
fix sonar
aomegax Jun 6, 2025
b6964ce
fix sonar
aomegax Jun 6, 2025
01ccbac
fix sonar
aomegax Jun 6, 2025
7026109
fix sonar
aomegax Jun 6, 2025
6c0ee40
fix
aomegax Jun 6, 2025
d58615a
fix sonar
aomegax Jun 6, 2025
d80f8e0
fix sonar
aomegax Jun 6, 2025
50f199a
fix
aomegax Jun 6, 2025
2fbd750
Bump to version 0.0.3-1-PAGOPA-2289-massive [skip ci]
pagopa-github-bot Jun 6, 2025
608db1e
fix infra
aomegax Jun 6, 2025
f35294a
Merge branch 'PAGOPA-2289-massive' of github.com:pagopa/pagopa-api-co…
aomegax Jun 6, 2025
e82378e
fix infra
aomegax Jun 6, 2025
f86615f
fix cd pipe
aomegax Jun 6, 2025
8603c8d
fix cd pipe
aomegax Jun 6, 2025
748be53
added * and */
aomegax Jun 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
504 changes: 303 additions & 201 deletions openapi/openapi.json
View file
Open in desktop

Large diffs are not rendered by default.

114 changes: 70 additions & 44 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,20 @@
<groupId>it.gov.pagopa.api-config</groupId>
<artifactId>testing-support</artifactId>
<version>0.0.3</version>
<description>Expose apis for config table crud,for TEST purposes only</description>
<description>Expose apis for config table crud, for TEST purposes only</description>

<properties>
<java.version>17</java.version>
<postgresql.version>42.5.4</postgresql.version>
<ojdbc8-version>21.4.0.0</ojdbc8-version>
</properties>

<repositories>
<repository>
<id>github</id>
<url>https://public:${env.GITHUB_TOKEN_READ_PACKAGES}@maven.pkg.github.com/pagopa/pagopa-api-config-starter</url>
</repository>
</repositories>
<!-- <repositories>-->
<!-- <repository>-->
<!-- <id>github</id>-->
<!-- <url>https://public:${env.GITHUB_TOKEN_READ_PACKAGES}@maven.pkg.github.com/pagopa/pagopa-api-config-starter</url>-->
<!-- </repository>-->
<!-- </repositories>-->

<dependencies>
<dependency>
Expand Down Expand Up @@ -128,49 +128,75 @@
</dependency>
</dependencies>

<profiles>
<profile>
<id>openapi</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<dependencies>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
</dependencies>
</profile>
<profile>
<id>sonar</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.7</version>
<configuration>
<excludes>
<exclude>**/imported/**</exclude>
<exclude>**/model/**</exclude>
<exclude>**/exception/**</exclude>
</excludes>
</configuration>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.9.1.2184</version>
<executions>
<execution>
<phase>verify</phase>
<goals>
<goal>sonar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.7</version>
<configuration>
<excludes>
</excludes>
</configuration>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonarsource.scanner.maven</groupId>
<artifactId>sonar-maven-plugin</artifactId>
<version>3.3.0.603</version>
<executions>
<execution>
<phase>verify</phase>
<goals>
<goal>sonar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<testResources>
<testResource>
Expand Down
63 changes: 42 additions & 21 deletions src/main/java/it/gov/pagopa/apiconfig/testingsupport/controller/GenericQueryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,16 @@

import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.ExampleObject;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import it.gov.pagopa.apiconfig.testingsupport.service.GenericQueryService;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.validation.constraints.NotBlank;

@RestController
public class GenericQueryController {
Expand All @@ -24,22 +22,45 @@ public class GenericQueryController {

@PostMapping("/genericQuery")
@Operation(
summary = "Executes the query.",
description = "This endpoint executes the query that is passed after being sanitized and then returns the result.",
requestBody = @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "Query to be executed.",
required = true,
content = @Content(
mediaType = "application/json",
schema = @Schema(type = "string", example = "SELECT * FROM TABLE")
)),
responses = {
@ApiResponse(responseCode = "200", description = "Success"),
@ApiResponse(responseCode = "400", description = "Bad Request"),
@ApiResponse(responseCode = "403", description = "Forbidden"),
@ApiResponse(responseCode = "500", description = "Internal Server Error")
})
summary = "Executes the query.",
description = "This endpoint executes the query that is passed after being sanitized and then returns the result.",
requestBody = @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "Query to be executed.",
required = true,
content = @Content(
mediaType = "application/json",
schema = @Schema(type = "string", example = "SELECT * FROM TABLE")
)),
responses = {
@ApiResponse(responseCode = "200", description = "Success"),
@ApiResponse(responseCode = "400", description = "Bad Request"),
@ApiResponse(responseCode = "403", description = "Forbidden"),
@ApiResponse(responseCode = "500", description = "Internal Server Error")
})
public ResponseEntity<List<Object>> getQueryResponse(@RequestBody String query){
return ResponseEntity.ok(service.getQueryResponse(query));
}

@PostMapping("/massive")
@Operation(
summary = "Executes the queries contained in the input file.",
description = "This endpoint executes the queries that is passed in a single transaction after being sanitized and then returns the result.",
requestBody = @io.swagger.v3.oas.annotations.parameters.RequestBody(
description = "Query to be executed.",
required = true,
content = @Content(
mediaType = "multipart/form-data",
schema = @Schema(type = "string", example = "file")
)),
responses = {
@ApiResponse(responseCode = "200", description = "Success"),
@ApiResponse(responseCode = "400", description = "Bad Request"),
@ApiResponse(responseCode = "403", description = "Forbidden"),
@ApiResponse(responseCode = "500", description = "Internal Server Error")
})
public ResponseEntity<List<Object>> getMassiveQueryResponse(
@RequestParam("file") @NotBlank MultipartFile file
){
return ResponseEntity.ok(service.getMassiveQueryResponse(file));
}
}
6 changes: 4 additions & 2 deletions src/main/java/it/gov/pagopa/apiconfig/testingsupport/exception/AppError.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ public enum AppError {
INTERNAL_SERVER_ERROR(HttpStatus.INTERNAL_SERVER_ERROR, "Internal Server Error",
"Something was wrong"),
DANGEROUS_QUERY(HttpStatus.FORBIDDEN, "Dangerous query detected.",
"Dangerous query detected, the query was not executed."),
WRONG_QUERY_GRAMMAR(HttpStatus.INTERNAL_SERVER_ERROR, "Something went wrong.",
"Dangerous query detected, the query was not executed."),
FILE_EMPTY(HttpStatus.BAD_REQUEST, "File empty.",
"File must be a valid sql file"),
WRONG_QUERY_GRAMMAR(HttpStatus.BAD_REQUEST, "Something went wrong.",
"Something went wrong with the query, wrong grammar or invalid table.");

public final HttpStatus httpStatus;
Expand Down
95 changes: 82 additions & 13 deletions src/main/java/it/gov/pagopa/apiconfig/testingsupport/service/GenericQueryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

import it.gov.pagopa.apiconfig.testingsupport.exception.AppError;
import it.gov.pagopa.apiconfig.testingsupport.exception.AppException;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import javax.persistence.EntityManager;
Expand All @@ -10,43 +16,87 @@
import javax.persistence.Query;
import org.hibernate.query.internal.NativeQueryImpl;
import org.hibernate.transform.AliasToEntityMapResultTransformer;
import org.postgresql.util.PSQLException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;

@Service
public class GenericQueryService {

@PersistenceContext
private EntityManager entityManager;
EntityManager entityManager;

@Value("${dangerous.sql.keywords}")
private List<String> dangerousKeywordsList;
List<String> dangerousKeywordsList;

@Autowired
private JdbcTemplate jdbcTemplate;
JdbcTemplate jdbcTemplate;

@Transactional
public List getQueryResponse(String query) {
String sanitizedQuery = sanitizeQuery(query);
try {
Query nquery = entityManager.createNativeQuery(sanitizedQuery);
NativeQueryImpl nativeQuery = (NativeQueryImpl) nquery;
nativeQuery.setResultTransformer(AliasToEntityMapResultTransformer.INSTANCE);
if(modifying(query)){
return Collections.singletonList(nativeQuery.executeUpdate());
return executeQuery(sanitizedQuery);
} catch (PersistenceException pExc) {
throw new AppException(AppError.WRONG_QUERY_GRAMMAR);
} catch (Exception e) {
throw new AppException(AppError.INTERNAL_SERVER_ERROR);
}
}

@Transactional
public List<Object> getMassiveQueryResponse(MultipartFile file) {
if (file.isEmpty()) {
throw new AppException(AppError.FILE_EMPTY);
}

List<String> queries = new ArrayList<>();
try {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(file.getInputStream()))) {
StringBuilder queryBuilder = new StringBuilder();
String line;

while ((line = reader.readLine()) != null) {
line = line.trim();

// Skip empty lines and SQL comments
if (line.isEmpty() || line.startsWith("--") || line.startsWith("//") || line.startsWith("/*")) {
continue;
}

queryBuilder.append(line).append(" ");

if (line.endsWith(";")) {
String query = queryBuilder.toString().trim();
queryBuilder.setLength(0); // reset builder

String sanitizedQuery = sanitizeQuery(query);
// Remove the trailing semicolon
queries.add(sanitizedQuery.substring(0, query.length() - 1));
}
}
}
else {
// return nativeQuery.getResultList();
// necessary to maintain the select column order
return jdbcTemplate.queryForList(query);
} catch (IOException e) {
throw new AppException(AppError.DANGEROUS_QUERY);
}

List results = new ArrayList();
try {
for (String query : queries) {
results.add(executeQuery(query));
}
} catch (PersistenceException pExc) {
} catch (PersistenceException | PSQLException | BadSqlGrammarException pExc) {
throw new AppException(AppError.WRONG_QUERY_GRAMMAR);
} catch (Exception e) {
throw new AppException(AppError.INTERNAL_SERVER_ERROR);
}

return results;
}

private String sanitizeQuery(String originalQuery) {
Expand All @@ -57,8 +107,27 @@
return originalQuery;
}

/**
* return true if query is an update or insert
* @param query
* @return
*/
private boolean modifying(String query){
String lowerCase = query.toLowerCase();
return lowerCase.contains("update") || lowerCase.contains("create table") || lowerCase.contains("insert into");
}

private List executeQuery(String query) throws Exception {
Query nquery = entityManager.createNativeQuery(query);
NativeQueryImpl nativeQuery = (NativeQueryImpl) nquery;
nativeQuery.setResultTransformer(AliasToEntityMapResultTransformer.INSTANCE);
if(modifying(query)){
return Collections.singletonList(nativeQuery.executeUpdate());
}
else {
// necessary to maintain the select column order
return jdbcTemplate.queryForList(query);
}
}

}
2 changes: 1 addition & 1 deletion src/main/resources/application-local.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ cors.configuration={"origins": ["*"], "methods": ["*"]}
#spring.datasource.username=postgres
#spring.datasource.password=pwd

spring.datasource.url=jdbc:postgresql://pagopa-d-weu-nodo-flexible-postgresql.postgres.database.azure.com:6432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg
spring.datasource.url=jdbc:postgresql://pagopa-d-weu-nodo-flexible-postgresql.postgres.database.azure.com:5432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg
#spring.datasource.url=jdbc:postgresql://pagopa-d-weu-nodo-flexible-postgresql.postgres.database.azure.com:6432/nodo-replica?sslmode=require&prepareThreshold=0&currentSchema=cfg
spring.datasource.username=cfg
spring.datasource.password=password
Expand Down
Loading
Loading