Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion .github/actions/setup-env/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,34 @@ runs:

- uses: actions/setup-node@v4
with:
node-version: 18.x
node-version: 18.x

- name: Aggressive disk cleanup
shell: bash
run: |
# Remove .NET SDKs
sudo rm -rf /usr/share/dotnet || true

# Remove Swift toolchain
sudo rm -rf /usr/share/swift || true

# Remove Haskell (GHC)
sudo rm -rf /usr/local/.ghcup || true

# Remove Julia
sudo rm -rf /usr/local/julia* || true

# Remove Android SDKs
sudo rm -rf /usr/local/lib/android || true

# Remove Chromium (optional if not using for browser tests)
sudo rm -rf /usr/local/share/chromium || true

# Remove Microsoft/Edge and Google Chrome builds
sudo rm -rf /opt/microsoft /opt/google || true

# Remove Azure CLI
sudo rm -rf /opt/az || true

# Remove PowerShell
sudo rm -rf /usr/local/share/powershell || true
11 changes: 11 additions & 0 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,12 @@ jobs:
path: |
**/build/jacoco/*.exec

- name: Show disk usage
if: always()
run: |
df -h
docker system df -v

jacoco-aggregate:
needs: [ gradle-tests ]
runs-on: ubuntu-latest
Expand Down Expand Up @@ -241,6 +247,11 @@ jobs:
- name: Run E2E test script
run: |
docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
- name: Show disk usage
if: always()
run: |
df -h
docker system df -v
- name: Collect Docker, OpenSearch Benchmark, and Shared Logs
if: always()
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sonar-qube.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:

services:
sonarqube:
image: sonarqube
image: sonarqube:25.10.0.114319-community
ports:
- 9000:9000
options: >-
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { App, Tags } from 'aws-cdk-lib';
import { readFileSync } from 'fs';
import { readFileSync } from 'node:fs';
import { StackComposer } from "../lib/stack-composer";

export function createApp(): App {
const app = new App();
const versionFile = readFileSync('../../../VERSION', 'utf-8');
const version = versionFile.replace(/\n/g, '');
const version = versionFile.replaceAll('\n', '');
Tags.of(app).add("migration_deployment", version);

const account = process.env.CDK_DEFAULT_ACCOUNT;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import {Secret} from "aws-cdk-lib/aws-secretsmanager";
import * as forge from 'node-forge';
import {ClusterYaml, SnapshotYaml} from "./migration-services-yaml";
import {CdkLogger} from "./cdk-logger";
import {mkdtempSync, writeFileSync} from 'fs';
import {join} from 'path';
import {tmpdir} from 'os';
import {execSync} from 'child_process';
import {mkdtempSync, writeFileSync} from 'node:fs';
import {join} from 'node:path';
import {tmpdir} from 'node:os';
import {execSync} from 'node:child_process';

export const MAX_IAM_ROLE_NAME_LENGTH = 64;
export const MAX_STAGE_NAME_LENGTH = 15;
Expand Down Expand Up @@ -188,7 +188,7 @@ export function createECSTaskRole(scope: Construct, serviceName: string, region:
const excessCharacters = taskRoleName.length - MAX_IAM_ROLE_NAME_LENGTH
if (excessCharacters > 0) {
if (excessCharacters > serviceName.length) {
throw Error(`Unexpected ECS task role name length for proposed name: '${taskRoleName}' could not be reasonably truncated
throw new Error(`Unexpected ECS task role name length for proposed name: '${taskRoleName}' could not be reasonably truncated
below ${MAX_IAM_ROLE_NAME_LENGTH} characters`)
}
const truncatedServiceName = serviceName.slice(0, serviceName.length - excessCharacters)
Expand Down Expand Up @@ -540,7 +540,7 @@ export function isRegionGovCloud(region: string): boolean {
* @returns {ContainerImage} - A `ContainerImage` object representing the Docker image asset.
*/
export function makeLocalAssetContainerImage(scope: Construct, imageName: string): ContainerImage {
const sanitizedImageName = imageName.replace(/[^a-zA-Z0-9-_]/g, '_');
const sanitizedImageName = imageName.replaceAll(/[^a-zA-Z0-9-_]/g, '_');
const tempDir = mkdtempSync(join(tmpdir(), 'docker-build-' + sanitizedImageName));
const dockerfilePath = join(tempDir, 'Dockerfile');

Expand All @@ -555,7 +555,7 @@ export function makeLocalAssetContainerImage(scope: Construct, imageName: string
if (!imageId) {
throw new Error(`No RepoDigests found for image: ${imageName}`);
}
imageHash = imageId.replace(/[^a-zA-Z0-9-_]/g, '_');
imageHash = imageId.replaceAll(/[^a-zA-Z0-9-_]/g, '_');
CdkLogger.info('For image: ' + imageName + ' found imageHash: ' + imageHash);
} catch (error) {
CdkLogger.error('Error fetching the actual hash for the image: ' + imageName + ' Error: ' + error);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
CloudFormationCustomResourceResponse,
} from 'aws-lambda';
import { ACMClient, ImportCertificateCommand, DeleteCertificateCommand } from '@aws-sdk/client-acm';
import * as https from 'https';
import * as https from 'node:https';
import * as forge from 'node-forge';

export const handler = async (event: CloudFormationCustomResourceEvent, context: Context): Promise<CloudFormationCustomResourceResponse> => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ export class VpcDetails {
return subnetType
}
}
throw Error(`Unable to find subnet ids: [${subnetIds}] in VPC: ${vpc.vpcId}. Please ensure all subnet ids exist and are of the same subnet type`)
throw new Error(`Unable to find subnet ids: [${subnetIds}] in VPC: ${vpc.vpcId}. Please ensure all subnet ids exist and are of the same subnet type`)
}

private validateProvidedSubnetIds(vpc: IVpc, vpcSubnetIds: string[], azCount: number) {
Expand All @@ -107,7 +107,7 @@ export class VpcDetails {
subnetFilters: [SubnetFilter.byIds(vpcSubnetIds)]
})
if (uniqueAzSubnets.subnetIds.length != vpcSubnetIds.length) {
throw Error(`Not all subnet ids provided: [${vpcSubnetIds}] were in a unique AZ`)
throw new Error(`Not all subnet ids provided: [${vpcSubnetIds}] were in a unique AZ`)
}
return uniqueAzSubnets
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Certificate, ICertificate } from "aws-cdk-lib/aws-certificatemanager";
import { Provider } from 'aws-cdk-lib/custom-resources';
import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
import { CustomResource, Duration, RemovalPolicy, Stack } from 'aws-cdk-lib/core';
import * as path from 'path';
import * as path from 'node:path';
import { Runtime } from 'aws-cdk-lib/aws-lambda';
import { PolicyStatement, Role, ServicePrincipal, ManagedPolicy } from 'aws-cdk-lib/aws-iam';
import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ export class MigrationServiceCore extends Stack {
logGroupName: `/migration/${props.stage}/${props.defaultDeployId}/${props.serviceName}`
});

const multilineRe = /^(\[[A-Z ]{1,5}\] )?\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}/;
const serviceContainer = serviceTaskDef.addContainer("ServiceContainer", {
image: serviceImage,
containerName: props.serviceName,
Expand All @@ -100,7 +101,7 @@ export class MigrationServiceCore extends Stack {
// and "[ERROR] 2024-12-31 23:59:59..."
// and "2024-12-31 23:59:59..."
// and "2024-12-31T23:59:59..."
multilinePattern: "^(\\[[A-Z ]{1,5}\\] )?\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}",
multilinePattern: multilineRe.source,
// Defer buffering behavior to log4j2 for greater flexibility
mode: AwsLogDriverMode.BLOCKING,
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
const extraArgsDict = parseArgsToDict(props.extraArgs)
const storagePath = "/storage"
const planningSize = props.maxShardSizeGiB ?? 80;
const planningSizeBuffer = 1.10
const planningSizeBuffer = 1.1
const maxShardSizeGiB = planningSize * planningSizeBuffer
const maxShardSizeBytes = maxShardSizeGiB * (1024 ** 3)
if (props.skipClusterCertCheck != false) { // when true or unspecified, add the flag
Expand Down Expand Up @@ -126,7 +126,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {

// Calculate the volume size based on the max shard size
// Have space for the snapshot and an unpacked copy, with buffer
const shardVolumeSizeGiBBufferMultiple = 1.10
const shardVolumeSizeGiBBufferMultiple = 1.1
const shardVolumeSizeGiB = Math.max(
Math.ceil(maxShardSizeGiB * 2 * shardVolumeSizeGiBBufferMultiple),
1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {Construct} from "constructs";
import {Duration, Stack, StackProps} from "aws-cdk-lib";
import {readFileSync} from 'fs';
import {readFileSync} from 'node:fs';
import {OpenSearchDomainStack} from "./opensearch-domain-stack";
import {EngineVersion, TLSSecurityPolicy} from "aws-cdk-lib/aws-opensearchservice";
import * as defaultValuesJson from "../default-values.json"
Expand Down Expand Up @@ -62,7 +62,7 @@ export class StackComposer {
// Values provided by the CLI will always be represented as a string and need to be parsed
if (typeof option === 'string') {
if (expectedType === 'number') {
return parseInt(option)
return Number.parseInt(option)
}
if (expectedType === 'boolean' || expectedType === 'object') {
try {
Expand All @@ -77,7 +77,7 @@ export class StackComposer {
}
// Values provided by the cdk.context.json should be of the desired type
if (typeof option !== expectedType) {
throw new Error(`Type provided by cdk.context.json for ${optionName} was ${typeof option} but expected ${expectedType}`)
throw new TypeError(`Type provided by cdk.context.json for ${optionName} was ${typeof option} but expected ${expectedType}`)
}
return option
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { App, Tags } from 'aws-cdk-lib';
import { createApp } from '../bin/createApp';
import { StackComposer } from '../lib/stack-composer';

jest.mock('fs', () => ({
jest.mock('node:fs', () => ({
readFileSync: jest.fn().mockReturnValue('1.0.0\n'),
}));

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

jest.mock('child_process', () => {
const child_process = jest.requireActual('child_process');
jest.mock('node:child_process', () => {
const child_process = jest.requireActual('node:child_process');

// Define the list of expected Docker images as per CI.yml
const expectedDockerImages = [
Expand Down
4 changes: 2 additions & 2 deletions deployment/cdk/opensearch-service-migration/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"compilerOptions": {
"outDir": "dist",
"target": "ES2020",
"target": "ES2021",
"module": "commonjs",
"lib": ["es2020"],
"lib": ["es2021"],
"resolveJsonModule": true,
"declaration": true,
"strict": true,
Expand Down
51 changes: 0 additions & 51 deletions frontend/__tests__/utils/jsonUtils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,55 +82,4 @@ describe("jsonUtils", () => {
expect(prettyPrintJson(input)).toBe(expected);
});
});

describe("readFileAsText", () => {
// Helper function to create a mock FileReader
const createMockFileReader = (content: string, shouldSucceed = true) => {
if (shouldSucceed) {
return {
onload: null as any,
onerror: null as any,
result: content,
readAsText: function() {
setTimeout(() => {
if (this.onload) this.onload();
}, 0);
}
};
} else {
const mockError = new Error("Failed to read file");
return {
onload: null as any,
onerror: null as any,
readAsText: function() {
setTimeout(() => {
if (this.onerror) this.onerror(mockError);
}, 0);
}
};
}
};

it("should read file content as text", async () => {
// Create a mock File object
const fileContent = '{"key": "value"}';
const file = new File([fileContent], "test.json", { type: "application/json" });

// Replace the global FileReader with our mock
global.FileReader = jest.fn(() => createMockFileReader(fileContent)) as any;

const result = await readFileAsText(file);
expect(result).toBe(fileContent);
});

it("should reject with error when file reading fails", async () => {
// Create a mock File object
const file = new File(["content"], "test.json", { type: "application/json" });

// Replace the global FileReader with our mock that fails
global.FileReader = jest.fn(() => createMockFileReader("", false)) as any;

await expect(readFileAsText(file)).rejects.toThrow("Failed to read file: test.json");
});
});
});
4 changes: 2 additions & 2 deletions frontend/eslint.config.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { dirname } from "path";
import { fileURLToPath } from "url";
import { dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { FlatCompat } from "@eslint/eslintrc";

const __filename = fileURLToPath(import.meta.url);
Expand Down
2 changes: 1 addition & 1 deletion frontend/next.config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { NextConfig } from "next";

const { execSync } = require('child_process');
const { execSync } = require('node:child_process');

function getGitMostRecentTag() {
return execOrUnknown('git describe --tags --abbrev=0 HEAD');
Expand Down
6 changes: 4 additions & 2 deletions frontend/src/components/common/WorkflowWizard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export default function WorkflowWizard({

// Initialize activeStepIndex from URL or default to 0
const stepParam = searchParams?.get("step");
const initialStep = stepParam ? parseInt(stepParam, 10) : 0;
const initialStep = stepParam ? Number.parseInt(stepParam, 10) : 0;

const [activeStepIndex, setActiveStepIndex] = useState(initialStep);

Expand All @@ -52,7 +52,9 @@ export default function WorkflowWizard({
const query = search ? `?${search}` : "";

// Update URL without refreshing the page
router.replace(`${window.location.pathname}${query}`, { scroll: false });
router.replace(`${globalThis.location.pathname}${query}`, {
scroll: false,
});
}, [activeStepIndex, sessionName, router, searchParams]);

const handleNavigate = ({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ export default function TransformationSection() {
dimensions: Record<string, { rowSpan: number }>,
) => {
const newDimensions = { ...dimensions };
transformations.forEach((transform) => {
for (const transform of transformations) {
if (!newDimensions[transform.id]) {
newDimensions[transform.id] = { rowSpan: 1 };
}
});
}
return newDimensions;
};

Expand All @@ -49,11 +49,12 @@ export default function TransformationSection() {
dimensions: Record<string, { rowSpan: number }>,
) => {
const newDimensions = { ...dimensions };
Object.keys(newDimensions).forEach((id) => {
const ids = Object.keys(newDimensions);
for (const id of ids) {
if (!transformations.some((t) => t.id === id)) {
delete newDimensions[id];
}
});
}
return newDimensions;
};

Expand Down
6 changes: 4 additions & 2 deletions frontend/src/components/snapshot/SnapshotForm.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,10 @@ export default function SnapshotConfigView({

if (snapshotConfig.source.type === "s3") {
const source = snapshotConfig.source as S3SnapshotSource;
configItems.push({ label: "S3 URI", value: source.uri });
configItems.push({ label: "AWS Region", value: source.region });
configItems.push(
{ label: "S3 URI", value: source.uri },
{ label: "AWS Region", value: source.region },
);
}
if (snapshotConfig.source.type === "filesytem") {
const source = snapshotConfig.source as FileSystemSnapshotSource;
Expand Down
Loading