Skip to content

Add JDBC URL option TC_COPY_FILE, copying files to container #6387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/modules/databases/jdbc.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,18 @@ public class JDBCDriverTest {
...
```

### Copying files to the container

You may copy files into the container by specifying `TC_COPY_FILES`.
This is useful for supplying custom configuration files or utilizing the entrypoint of a Docker image to run scripts.

The syntax is: `?TC_COPY_FILES=host-path:container-path[:file-mode]` (file mode is optional).
If the host path starts with `/` it will be considered an absolute path, otherwise it's mapped from a classpath resource.

* Single file: `jdbc:tc:mysql:5.7.34:///databasename?TC_COPY_FILES=some-path/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql`
* Single file with file mode: `jdbc:tc:mysql:5.7.34:///databasename?TC_COPY_FILES=some-path/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql:755`
* Multiple files: `jdbc:tc:mysql:5.7.34:///databasename?TC_COPY_FILES=some-path/init_mysql.sql:/docker-entrypoint-initdb.d/init_mysql.sql:755,/absolute-path/another-file:/another/container-path/another-file`

### Running container in daemon mode

By default database container is being stopped as soon as last connection is closed. There are cases when you might need to start container and keep it running till you stop it explicitly or JVM is shutdown. To do this, add `TC_DAEMON` parameter to the URL as follows:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public class AbstractJDBCDriverTest {
protected enum Options {
ScriptedSchema,
CharacterSet,
CopyFiles,
CustomIniFile,
JDBCParams,
PmdKnownBroken,
Expand Down Expand Up @@ -63,7 +64,7 @@ public void test() throws SQLException {
performTestForCharacterEncodingForInitialScriptConnection(dataSource);
}

if (options.contains(Options.CustomIniFile)) {
if (options.contains(Options.CopyFiles) || options.contains(Options.CustomIniFile)) {
performTestForCustomIniFile(dataSource);
}
}
Expand Down Expand Up @@ -204,13 +205,13 @@ private HikariDataSource verifyCharacterSet(String jdbcUrl) throws SQLException
private void performTestForCustomIniFile(HikariDataSource dataSource) throws SQLException {
assumeFalse(SystemUtils.IS_OS_WINDOWS);
Statement statement = dataSource.getConnection().createStatement();
statement.execute("SELECT @@GLOBAL.innodb_file_format");
statement.execute("SELECT @@GLOBAL.key_buffer_size");
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Barracuda was already the default value for innodb_file_format, thus not actually testing the config was applied. I doubled the value of key_buffer_size in the ini file and asserted based on that instead.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might be the reason why the mariadb tests fail.

ResultSet resultSet = statement.getResultSet();

assertThat(resultSet.next()).as("The query returns a result").isTrue();
String result = resultSet.getString(1);

assertThat(result).as("The InnoDB file format has been set by the ini file content").isEqualTo("Barracuda");
assertThat(result).as("The InnoDB file format has been set by the ini file content").isEqualTo("32768");
}

private HikariDataSource getDataSource(String jdbcUrl, int poolSize) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@
import lombok.EqualsAndHashCode;
import lombok.Getter;
import org.testcontainers.UnstableAPI;
import org.testcontainers.utility.MountableFile;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand Down Expand Up @@ -61,6 +65,8 @@ public class ConnectionUrl {

private Map<String, String> tmpfsOptions = new HashMap<>();

private Map<String, List<MountableFile>> copyFilesToContainerOptions = Collections.emptyMap();

public static ConnectionUrl newInstance(final String url) {
ConnectionUrl connectionUrl = new ConnectionUrl(url);
connectionUrl.parseUrl();
Expand Down Expand Up @@ -135,6 +141,8 @@ private void parseUrl() {

tmpfsOptions = parseTmpfsOptions(containerParameters);

copyFilesToContainerOptions = parseCopyFilesToContainerOptions(containerParameters);

initScriptPath = Optional.ofNullable(containerParameters.get("TC_INITSCRIPT"));

reusable = Boolean.parseBoolean(containerParameters.get("TC_REUSABLE"));
Expand All @@ -160,6 +168,31 @@ private Map<String, String> parseTmpfsOptions(Map<String, String> containerParam
.collect(Collectors.toMap(string -> string.split(":")[0], string -> string.split(":")[1]));
}

private Map<String, List<MountableFile>> parseCopyFilesToContainerOptions(Map<String, String> containerParameters) {
if (!containerParameters.containsKey("TC_COPY_FILES")) {
return Collections.emptyMap();
}

String copyFilesToContainerOptions = containerParameters.get("TC_COPY_FILES");

Map<String, List<MountableFile>> mounts = new HashMap<>();
Arrays
.stream(copyFilesToContainerOptions.split(","))
.map(string -> string.split(":"))
.forEach(split -> {
String hostPath = split[0];
String containerPath = split[1];
Integer mode = split.length > 2 ? Integer.parseInt(split[2], 8) : null;
MountableFile mountableFile = (hostPath.startsWith("/"))
? MountableFile.forHostPath(hostPath, mode)
: MountableFile.forClasspathResource(hostPath, mode);
mounts.putIfAbsent(containerPath, new ArrayList<>());
mounts.get(containerPath).add(mountableFile);
});

return mounts;
}

/**
* Get the TestContainers Parameters such as Init Function, Init Script path etc.
*
Expand Down Expand Up @@ -201,6 +234,10 @@ public Map<String, String> getTmpfsOptions() {
return Collections.unmodifiableMap(tmpfsOptions);
}

public Map<String, List<MountableFile>> getCopyFilesToContainerOptions() {
return Collections.unmodifiableMap(copyFilesToContainerOptions);
}

/**
* This interface defines the Regex Patterns used by {@link ConnectionUrl}.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,15 @@ public synchronized Connection connect(String url, final Properties info) throws
if (candidateContainerType.supports(connectionUrl.getDatabaseType())) {
container = candidateContainerType.newInstance(connectionUrl);
container.withTmpFs(connectionUrl.getTmpfsOptions());

JdbcDatabaseContainer finalContainer = container;
connectionUrl
.getCopyFilesToContainerOptions()
.forEach((containerPath, mountableFiles) -> {
mountableFiles.forEach(mountableFile -> {
finalContainer.withCopyFileToContainer(mountableFile, containerPath);
});
});
delegate = container.getJdbcDriverInstance();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.testcontainers.utility.MountableFile;

import static org.assertj.core.api.Assertions.assertThat;

Expand Down Expand Up @@ -68,6 +69,62 @@ public void testTmpfsOption() {
assertThat(url.getTmpfsOptions()).as("tmpfs option key1 has correct value").containsEntry("key1", "value1");
}

@Test
public void testCopyFilesOption() {
String urlString =
"jdbc:tc:mysql://somehostname/databasename?TC_COPY_FILES=/key:path1,logback-test.xml:path2:755";
ConnectionUrl url = ConnectionUrl.newInstance(urlString);

assertThat(url.getQueryParameters()).as("Connection Parameters set is empty").isEmpty();
assertThat(url.getContainerParameters()).as("Container Parameters set is not empty").isNotEmpty();
assertThat(url.getContainerParameters())
.as("Container Parameter TC_COPY_FILES is true")
.containsEntry("TC_COPY_FILES", "/key:path1,logback-test.xml:path2:755");
assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option has correct values")
.containsOnlyKeys("path1", "path2");

assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path1")
.asList()
.hasSize(1);
assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path1")
.asList()
.element(0)
.extracting("filesystemPath")
.isEqualTo("/key");
assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path1")
.asList()
.element(0)
.extracting("fileMode")
.isEqualTo(0100000 | 0644);

assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path2")
.asList()
.hasSize(1);
assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path2")
.asList()
.element(0)
.extracting("filesystemPath")
.isEqualTo(MountableFile.forClasspathResource("logback-test.xml").getFilesystemPath());
assertThat(url.getCopyFilesToContainerOptions())
.as("copyFiles option path1 has correct value")
.extractingByKey("path2")
.asList()
.element(0)
.extracting("fileMode")
.isEqualTo(0100000 | 0755);
}

@Test
public void testInitScriptPathCapture() {
String urlString =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ public static Iterable<Object[]> data() {
"jdbc:tc:mysql:5.6.51://hostname/databasename?TC_MY_CNF=somepath/mysql_conf_override",
EnumSet.of(Options.CustomIniFile),
},
{
"jdbc:tc:mysql://hostname/databasename?TC_COPY_FILES=somepath/mysql_conf_override/my.cnf:/etc/mysql/conf.d/mysqld.cnf",
EnumSet.of(Options.CopyFiles),
},
}
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ user = mysql
port = 3306
#socket = /tmp/mysql.sock
skip-external-locking
key_buffer_size = 16K
key_buffer_size = 32K
max_allowed_packet = 1M
table_open_cache = 4
sort_buffer_size = 64K
Expand Down Expand Up @@ -51,4 +51,4 @@ innodb_buffer_pool_size = 16M
innodb_log_file_size = 5M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 1
innodb_lock_wait_timeout = 50
innodb_lock_wait_timeout = 50