Skip to content

Commit c9852d9

Browse files
azasypkinazasypkin
authored
feat(security,uiam,cps): update @kbn/es defaults to support CPS-enabled local projects (elastic#246061)
## Summary Update `@kbn/es` defaults to support CPS-enabled local projects. ## How to test Run both Elasticsearch and Kibana in CPS mode: ```bash # Run Elasticsearch Serverless $ yarn es serverless --projectType observability --uiam \ -E serverless.cross_project.enabled=true # Run Kibana Serverless $ yarn start --serverless=oblt --uiam \ --cps.enabled=true --cps.cpsEnabled=true ``` Check local project tags with Dev Tools: ```http GET /_project/tags --- { "origin": { "abcde1234567890": { "_alias": "local_project", "_id": "abcde1234567890", "_organization": "org1234567890", "_type": "observability", "env": "local" } } } ``` or with `cURL`: ```bash curl -ku elastic_serverless:changeme -X GET 'https://localhost:9200/_project/tags?pretty' --- { "origin" : { "abcde1234567890" : { "_alias" : "local_project", "_id" : "abcde1234567890", "_organization" : "org1234567890", "_type" : "observability", "env" : "local" } } } ``` /cc @mbondyra
1 parent c44d07f commit c9852d9

7 files changed

Lines changed: 74 additions & 17 deletions

File tree

‎src/platform/packages/shared/kbn-es/src/paths.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ export const SERVERLESS_JWKS_PATH = resolve(__dirname, './serverless_resources/j
6060

6161
export const SERVERLESS_IDP_METADATA_PATH = resolve(REPO_ROOT, '.es', 'idp_metadata.xml');
6262

63+
export const SERVERLESS_OPERATOR_PATH = resolve(REPO_ROOT, '.es', 'operator');
64+
6365
export const SERVERLESS_UIAM_ENTRYPOINT_PATH = resolve(
6466
__dirname,
6567
'./serverless_resources/run_java_with_custom_ca.sh'

‎src/platform/packages/shared/kbn-es/src/serverless_resources/secrets.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
},
66
"string_secrets": {
77
"xpack.security.transport.ssl.keystore.secure_password": "storepass",
8+
"xpack.security.remote_cluster_client.ssl.keystore.secure_password": "storepass",
9+
"xpack.security.remote_cluster_server.ssl.keystore.secure_password": "storepass",
810
"xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret": "my_super_secret",
911
"serverless.universal_iam_service.shared_secret": "Dw7eRt5yU2iO9pL3aS4dF6gH8jK0lZ1xC2vB3nM4qW5="
1012
}

‎src/platform/packages/shared/kbn-es/src/serverless_resources/secrets_ssl.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
"string_secrets": {
77
"xpack.security.http.ssl.keystore.secure_password": "storepass",
88
"xpack.security.transport.ssl.keystore.secure_password": "storepass",
9+
"xpack.security.remote_cluster_client.ssl.keystore.secure_password": "storepass",
10+
"xpack.security.remote_cluster_server.ssl.keystore.secure_password": "storepass",
911
"xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret": "my_super_secret",
1012
"serverless.universal_iam_service.shared_secret": "Dw7eRt5yU2iO9pL3aS4dF6gH8jK0lZ1xC2vB3nM4qW5="
1113
}

‎src/platform/packages/shared/kbn-es/src/utils/docker.test.ts‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import {
4242
SERVERLESS_SECRETS_PATH,
4343
SERVERLESS_JWKS_PATH,
4444
SERVERLESS_IDP_METADATA_PATH,
45+
SERVERLESS_OPERATOR_PATH,
4546
} from '../paths';
4647
import * as waitClusterUtil from './wait_until_cluster_ready';
4748
import * as waitForSecurityIndexUtil from './wait_for_security_index';
@@ -117,15 +118,16 @@ const serverlessResources = SERVERLESS_RESOURCES_PATHS.reduce<string[]>((acc, pa
117118
}, []);
118119

119120
const volumeCmdTest = async (volumeCmd: string[]) => {
120-
expect(volumeCmd).toHaveLength(22);
121+
expect(volumeCmd).toHaveLength(24);
121122
expect(volumeCmd).toEqual(
122123
expect.arrayContaining([
123124
...getESp12Volume(),
124125
...serverlessResources,
125126
`${baseEsPath}:/objectstore:z`,
126127
`stateless.object_store.bucket=${serverlessDir}`,
127128
`${SERVERLESS_SECRETS_PATH}:${SERVERLESS_CONFIG_PATH}secrets/secrets.json:z`,
128-
`${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}secrets/jwks.json:z`,
129+
`${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}jwks/jwks.json:z`,
130+
`${SERVERLESS_OPERATOR_PATH}:${SERVERLESS_CONFIG_PATH}operator`,
129131
])
130132
);
131133

@@ -480,7 +482,7 @@ describe('resolveEsArgs()', () => {
480482
"--env",
481483
"xpack.security.authc.realms.saml.cloud-saml-kibana.order=0",
482484
"--env",
483-
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/secrets/idp_metadata.xml",
485+
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/idp_metadata.xml",
484486
"--env",
485487
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=urn:mock-idp",
486488
"--env",
@@ -547,7 +549,7 @@ describe('resolveEsArgs()', () => {
547549
"--env",
548550
"xpack.security.authc.realms.saml.cloud-saml-kibana.order=0",
549551
"--env",
550-
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/secrets/idp_metadata.xml",
552+
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/idp_metadata.xml",
551553
"--env",
552554
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=urn:mock-idp",
553555
"--env",
@@ -594,7 +596,7 @@ describe('resolveEsArgs()', () => {
594596
"--env",
595597
"xpack.security.authc.realms.saml.cloud-saml-kibana.order=0",
596598
"--env",
597-
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/secrets/idp_metadata.xml",
599+
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=/usr/share/elasticsearch/config/idp_metadata.xml",
598600
"--env",
599601
"xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=urn:mock-idp",
600602
"--env",
@@ -666,7 +668,7 @@ describe('setupServerlessVolumes()', () => {
666668
basePath: baseEsPath,
667669
});
668670

669-
volumeCmdTest(volumeCmd);
671+
await volumeCmdTest(volumeCmd);
670672
await expect(Fsp.access(serverlessObjectStorePath)).resolves.not.toThrow();
671673
});
672674

@@ -675,7 +677,7 @@ describe('setupServerlessVolumes()', () => {
675677

676678
const volumeCmd = await setupServerlessVolumes(log, { projectType, basePath: baseEsPath });
677679

678-
volumeCmdTest(volumeCmd);
680+
await volumeCmdTest(volumeCmd);
679681
await expect(
680682
Fsp.access(`${serverlessObjectStorePath}/cluster_state/lease`)
681683
).resolves.not.toThrow();
@@ -690,7 +692,7 @@ describe('setupServerlessVolumes()', () => {
690692
clean: true,
691693
});
692694

693-
volumeCmdTest(volumeCmd);
695+
await volumeCmdTest(volumeCmd);
694696
await expect(
695697
Fsp.access(`${serverlessObjectStorePath}/cluster_state/lease`)
696698
).rejects.toThrowError();
@@ -719,7 +721,7 @@ describe('setupServerlessVolumes()', () => {
719721
const pathsNotIncludedInCmd = requiredPaths.filter(
720722
(path) => !volumeCmd.some((cmd) => cmd.includes(path))
721723
);
722-
expect(volumeCmd).toHaveLength(24);
724+
expect(volumeCmd).toHaveLength(26);
723725
expect(pathsNotIncludedInCmd).toEqual([]);
724726
});
725727

‎src/platform/packages/shared/kbn-es/src/utils/docker.ts‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
SERVERLESS_FILES_PATH,
5151
SERVERLESS_SECRETS_SSL_PATH,
5252
SERVERLESS_ROLES_ROOT_PATH,
53+
SERVERLESS_OPERATOR_PATH,
5354
} from '../paths';
5455
import {
5556
ELASTIC_SERVERLESS_SUPERUSER,
@@ -266,7 +267,7 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [
266267

267268
[
268269
'xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path',
269-
`${SERVERLESS_CONFIG_PATH}secrets/jwks.json`,
270+
`${SERVERLESS_CONFIG_PATH}jwks/jwks.json`,
270271
],
271272

272273
['xpack.security.operator_privileges.enabled', 'true'],
@@ -279,6 +280,19 @@ const DEFAULT_SERVERLESS_ESARGS: Array<[string, string]> = [
279280
],
280281

281282
['xpack.security.transport.ssl.verification_mode', 'certificate'],
283+
284+
[
285+
'xpack.security.remote_cluster_client.ssl.keystore.path',
286+
`${SERVERLESS_CONFIG_PATH}certs/elasticsearch.p12`,
287+
],
288+
['xpack.security.remote_cluster_client.ssl.verification_mode', 'certificate'],
289+
290+
[
291+
'xpack.security.remote_cluster_server.ssl.keystore.path',
292+
`${SERVERLESS_CONFIG_PATH}certs/elasticsearch.p12`,
293+
],
294+
['xpack.security.remote_cluster_server.ssl.verification_mode', 'certificate'],
295+
['xpack.security.remote_cluster_server.ssl.client_authentication', 'required'],
282296
];
283297
// Temporary workaround for https://github.com/elastic/elasticsearch/issues/118583
284298
if (process.arch === 'arm64') {
@@ -625,7 +639,7 @@ export function resolveEsArgs(
625639
esArgs.set(`xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.order`, '0');
626640
esArgs.set(
627641
`xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.metadata.path`,
628-
`${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml`
642+
`${SERVERLESS_CONFIG_PATH}idp_metadata.xml`
629643
);
630644
esArgs.set(
631645
`xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.entity_id`,
@@ -859,20 +873,21 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
859873
await Fsp.writeFile(SERVERLESS_IDP_METADATA_PATH, metadata);
860874
volumeCmds.push(
861875
'--volume',
862-
`${SERVERLESS_IDP_METADATA_PATH}:${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml:z`
876+
`${SERVERLESS_IDP_METADATA_PATH}:${SERVERLESS_CONFIG_PATH}idp_metadata.xml:z`
863877
);
864878
}
865879

866880
volumeCmds.push(
867881
...getESp12Volume(),
868882
...serverlessResources,
883+
...(await getOperatorVolume(esProjectTypeFromKbn.get(projectType)!)),
869884

870885
'--volume',
871886
`${
872887
ssl ? SERVERLESS_SECRETS_SSL_PATH : SERVERLESS_SECRETS_PATH
873888
}:${SERVERLESS_CONFIG_PATH}secrets/secrets.json:z`,
874889
'--volume',
875-
`${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}secrets/jwks.json:z`
890+
`${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}jwks/jwks.json:z`
876891
);
877892

878893
return volumeCmds;
@@ -1125,3 +1140,37 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions
11251140
stdio: ['ignore', 'inherit', 'inherit'],
11261141
});
11271142
}
1143+
1144+
/**
1145+
* A volume mount for the operator folder, that contains operator specific configuration files like settings.json.
1146+
* We mount entire folder since Elasticsearch cannot properly watch changes in bind-mounted files.
1147+
* @param projectType Type of the serverless project.
1148+
*/
1149+
async function getOperatorVolume(projectType: string) {
1150+
await Fsp.mkdir(SERVERLESS_OPERATOR_PATH, { recursive: true });
1151+
1152+
// Settings should include information about the project that's normally populated by the Elasticsearch Controller.
1153+
const projectInfo = {
1154+
id: MOCK_IDP_UIAM_PROJECT_ID,
1155+
type: projectType,
1156+
alias: 'local_project',
1157+
organization: MOCK_IDP_UIAM_ORGANIZATION_ID,
1158+
};
1159+
const projectTags = {
1160+
...Object.fromEntries(Object.entries(projectInfo).map(([key, value]) => [`_${key}`, value])),
1161+
env: 'local',
1162+
};
1163+
1164+
await Fsp.writeFile(
1165+
join(SERVERLESS_OPERATOR_PATH, 'settings.json'),
1166+
JSON.stringify(
1167+
{
1168+
metadata: { version: '100', compatibility: '' },
1169+
state: { project: { ...projectInfo, tags: projectTags } },
1170+
},
1171+
null,
1172+
2
1173+
)
1174+
);
1175+
return ['--volume', `${SERVERLESS_OPERATOR_PATH}:${SERVERLESS_CONFIG_PATH}operator`];
1176+
}

‎src/platform/packages/shared/kbn-es/src/utils/docker_uiam.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe(`#runUiamContainer()`, () => {
8080
"curl -sk http://127.0.0.1:8080/ready | grep -q \\"\\\\\\"overall\\\\\\": true\\"",
8181
"--name",
8282
"uiam-cosmosdb",
83-
"mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-EN20251124",
83+
"mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-EN20251205",
8484
"--protocol",
8585
"https",
8686
"--port",
@@ -183,7 +183,7 @@ describe(`#runUiamContainer()`, () => {
183183
"timeout 1 bash -c \\"</dev/tcp/localhost/8080\\"",
184184
"--name",
185185
"uiam",
186-
"docker.elastic.co/cloud-ci/uiam:git-fd2a53b8cf9f",
186+
"docker.elastic.co/cloud-ci/uiam:git-f56eb1a3b9a8",
187187
],
188188
],
189189
Array [

‎src/platform/packages/shared/kbn-es/src/utils/docker_uiam.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ const COSMOS_DB_EMULATOR_DOCKER_REPO = `${COSMOS_DB_EMULATOR_DOCKER_REGISTRY}/co
2222

2323
// Check new version at https://github.com/Azure/azure-cosmos-db-emulator-docker/releases. DON'T use the rolling
2424
// `vnext-preview` image tag.
25-
const COSMOS_DB_EMULATOR_DOCKER_LATEST_VERIFIED_TAG = 'vnext-EN20251124';
25+
const COSMOS_DB_EMULATOR_DOCKER_LATEST_VERIFIED_TAG = 'vnext-EN20251205';
2626
export const COSMOS_DB_EMULATOR_DEFAULT_IMAGE = `${COSMOS_DB_EMULATOR_DOCKER_REPO}:${COSMOS_DB_EMULATOR_DOCKER_LATEST_VERIFIED_TAG}`;
2727

2828
const UIAM_DOCKER_REGISTRY = 'docker.elastic.co';
2929
const UIAM_DOCKER_REPO = `${UIAM_DOCKER_REGISTRY}/cloud-ci/uiam`;
3030
// Taken from GitOps version file for UIAM service (dev env, services/uiam/versions.yaml)
31-
const UIAM_DOCKER_LATEST_VERIFIED_TAG = 'git-fd2a53b8cf9f';
31+
const UIAM_DOCKER_LATEST_VERIFIED_TAG = 'git-f56eb1a3b9a8';
3232
export const UIAM_DEFAULT_IMAGE = `${UIAM_DOCKER_REPO}:${UIAM_DOCKER_LATEST_VERIFIED_TAG}`;
3333

3434
const UIAM_COSMOS_DB_NAME = 'uiam-db';

0 commit comments

Comments
 (0)