From 6f880f6ef5294adff11b4c8b0511108a9a44f6e5 Mon Sep 17 00:00:00 2001 From: Divyam Talwar Date: Mon, 29 Jun 2026 21:48:44 +0530 Subject: [PATCH] fix: expose TypeScript package subpaths The TypeScript docs reference package subpaths that were not present in the export map, so consumers following those examples could not rely on package-level resolution for metrics, models, kebab-case test cases, or evaluate helpers. This also aligns the existing annotation type subpath with a runtime export. Constraint: Keep the change additive and limited to documented or already-advertised TypeScript entrypoints. Rejected: Renaming or removing the existing testCase subpath | It may already be used by package consumers. Confidence: high Scope-risk: narrow Directive: Keep future public TypeScript subpaths covered by package export smoke tests. Tested: cd typescript && npm test -- package-exports.test.ts; cd typescript && npm run test:package-exports; cd typescript && npm run lint; temporary external TypeScript consumer compile check; git diff --check origin/main...HEAD Not-tested: Full cd typescript && npm test -- --runInBand passes locally | existing suite requires missing OPENAI_API_KEY and CONFIDENT_API_KEY, has an optional OpenAI Agents MCP dependency resolution failure, and includes an existing bad import in test/test-core/evaluate.test.ts. --- .github/workflows/typescript_test.yml | 8 + typescript/package.json | 38 +++ .../test/test-core/package-exports-smoke.cjs | 264 ++++++++++++++++++ .../test/test-core/package-exports.test.ts | 74 +++++ 4 files changed, 384 insertions(+) create mode 100644 typescript/test/test-core/package-exports-smoke.cjs create mode 100644 typescript/test/test-core/package-exports.test.ts diff --git a/.github/workflows/typescript_test.yml b/.github/workflows/typescript_test.yml index 5c91e8de5e..af9f5916af 100644 --- a/.github/workflows/typescript_test.yml +++ b/.github/workflows/typescript_test.yml @@ -13,6 +13,9 @@ on: - '.github/workflows/typescript_test.yml' workflow_dispatch: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -26,6 +29,7 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies + id: install run: npm ci - name: Run Jest tests @@ -34,3 +38,7 @@ jobs: CONFIDENT_API_KEY: ${{ secrets.CONFIDENT_API_KEY }} CONFIDENT_TRACE_VERBOSE: 0 run: npm test + + - name: Validate package exports + if: ${{ always() && steps.install.outcome == 'success' }} + run: npm run test:package-exports diff --git a/typescript/package.json b/typescript/package.json index c582a1c4d9..e2986d7a27 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -18,16 +18,41 @@ "require": "./dist/index.js", "types": "./dist/index.d.ts" }, + "./annotation": { + "import": "./dist/annotation/index.js", + "require": "./dist/annotation/index.js", + "types": "./dist/annotation/index.d.ts" + }, "./dataset": { "import": "./dist/dataset/index.js", "require": "./dist/dataset/index.js", "types": "./dist/dataset/index.d.ts" }, + "./metrics": { + "import": "./dist/metrics/index.js", + "require": "./dist/metrics/index.js", + "types": "./dist/metrics/index.d.ts" + }, + "./models": { + "import": "./dist/models/index.js", + "require": "./dist/models/index.js", + "types": "./dist/models/index.d.ts" + }, "./testCase": { "import": "./dist/test-case/index.js", "require": "./dist/test-case/index.js", "types": "./dist/test-case/index.d.ts" }, + "./test-case": { + "import": "./dist/test-case/index.js", + "require": "./dist/test-case/index.js", + "types": "./dist/test-case/index.d.ts" + }, + "./evaluate": { + "import": "./dist/evaluate/index.js", + "require": "./dist/evaluate/index.js", + "types": "./dist/evaluate/index.d.ts" + }, "./tracing": { "import": "./dist/tracing/index.js", "require": "./dist/tracing/index.js", @@ -72,9 +97,21 @@ "dataset": [ "dist/dataset/index.d.ts" ], + "metrics": [ + "dist/metrics/index.d.ts" + ], + "models": [ + "dist/models/index.d.ts" + ], "testCase": [ "dist/test-case/index.d.ts" ], + "test-case": [ + "dist/test-case/index.d.ts" + ], + "evaluate": [ + "dist/evaluate/index.d.ts" + ], "tracing": [ "dist/tracing/index.d.ts" ], @@ -107,6 +144,7 @@ "scripts": { "build": "tsc", "test": "jest", + "test:package-exports": "npm run build && node test/test-core/package-exports-smoke.cjs", "lint": "eslint", "lint:fix": "eslint --fix" }, diff --git a/typescript/test/test-core/package-exports-smoke.cjs b/typescript/test/test-core/package-exports-smoke.cjs new file mode 100644 index 0000000000..739af1fcd5 --- /dev/null +++ b/typescript/test/test-core/package-exports-smoke.cjs @@ -0,0 +1,264 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { execFileSync } = require("child_process"); + +const packageRoot = path.resolve(__dirname, "../.."); +const packageJson = require(path.join(packageRoot, "package.json")); + +const packageSubpaths = Object.keys(packageJson.exports) + .filter((exportKey) => exportKey !== ".") + .map((exportKey) => ({ + exportKey, + specifier: `deepeval/${exportKey.slice(2)}`, + })); + +const loadablePackageSubpaths = [ + "deepeval/annotation", + "deepeval/metrics", + "deepeval/models", + "deepeval/test-case", + "deepeval/evaluate", + "deepeval/testCase", +]; + +const privateSubpaths = ["deepeval/annotation/utils"]; + +const optionalPeerSubpaths = [ + { + specifier: "deepeval/integrations/langchain", + missingPeerSpecifier: "@langchain/core", + missingPeerPattern: /@langchain\/core/, + }, +]; + +const optionalPeerSpecifiers = new Set( + optionalPeerSubpaths.map(({ specifier }) => specifier), +); + +const consumerLoadableSubpaths = packageSubpaths + .map(({ specifier }) => specifier) + .filter((specifier) => !optionalPeerSpecifiers.has(specifier)); + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function formatSubpaths(subpaths) { + return subpaths.length > 0 ? subpaths.join(", ") : ""; +} + +function assertExportFilesExist(exportKey) { + const exportEntry = packageJson.exports[exportKey]; + assert(exportEntry, `Missing package export ${exportKey}`); + + for (const field of ["import", "require", "types"]) { + const relativePath = exportEntry[field]; + assert( + fs.existsSync(path.resolve(packageRoot, relativePath)), + `Missing ${field} file for ${exportKey}: ${relativePath}`, + ); + } +} + +function run(command, args, options = {}) { + execFileSync(command, args, { + stdio: "pipe", + encoding: "utf8", + ...options, + }); +} + +function loadSpecifier(specifier) { + require(specifier); + return import(specifier); +} + +async function assertOptionalPeerImport(specifier, missingPeerPattern) { + try { + await loadSpecifier(specifier); + } catch (error) { + assert( + error.code === "MODULE_NOT_FOUND" && + missingPeerPattern.test(error.message), + `${specifier} failed for an unexpected reason: ${error.message}`, + ); + return; + } +} + +function createConsumerScript() { + return ` +const assert = (condition, message) => { + if (!condition) throw new Error(message); +}; + +const loadableSubpaths = ${JSON.stringify(consumerLoadableSubpaths, null, 2)}; +const optionalPeerSubpaths = ${JSON.stringify( + optionalPeerSubpaths.map( + ({ specifier, missingPeerSpecifier, missingPeerPattern }) => ({ + specifier, + missingPeerSpecifier, + missingPeerPattern: missingPeerPattern.source, + }), + ), + null, + 2, + )}; + +(async () => { + for (const specifier of loadableSubpaths) { + require.resolve(specifier); + require(specifier); + await import(specifier); + } + + for (const { specifier, missingPeerSpecifier, missingPeerPattern } of optionalPeerSubpaths) { + require.resolve(specifier); + + let peerResolveError; + try { + require.resolve(missingPeerSpecifier); + } catch (error) { + peerResolveError = error; + } + + assert( + peerResolveError && peerResolveError.code === "MODULE_NOT_FOUND", + missingPeerSpecifier + " should not be installed in the packed consumer smoke test", + ); + + let commonJsError; + try { + require(specifier); + } catch (error) { + commonJsError = error; + } + + const expectedMissingPeer = new RegExp(missingPeerPattern); + assert( + commonJsError && + commonJsError.code === "MODULE_NOT_FOUND" && + expectedMissingPeer.test(commonJsError.message), + specifier + " should fail with missing optional peer " + missingPeerSpecifier, + ); + + let esmError; + try { + await import(specifier); + } catch (error) { + esmError = error; + } + + assert( + esmError && + esmError.code === "MODULE_NOT_FOUND" && + expectedMissingPeer.test(esmError.message), + specifier + " ESM import should fail with missing optional peer " + missingPeerSpecifier, + ); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); +`; +} + +function assertPackedArtifactConsumerInstall() { + const packDir = fs.mkdtempSync(path.join(os.tmpdir(), "deepeval-pack-")); + const consumerDir = fs.mkdtempSync( + path.join(os.tmpdir(), "deepeval-consumer-"), + ); + + try { + const tarball = execFileSync( + "npm", + ["pack", "--pack-destination", packDir, "--silent"], + { cwd: packageRoot, encoding: "utf8" }, + ).trim(); + const tarballPath = path.join(packDir, tarball); + + fs.writeFileSync( + path.join(consumerDir, "package.json"), + JSON.stringify({ private: true }, null, 2), + ); + + run( + "npm", + [ + "install", + "--ignore-scripts", + "--omit=dev", + "--no-audit", + "--no-fund", + tarballPath, + ], + { cwd: consumerDir }, + ); + + run("node", ["-e", createConsumerScript()], { cwd: consumerDir }); + } finally { + fs.rmSync(packDir, { recursive: true, force: true }); + fs.rmSync(consumerDir, { recursive: true, force: true }); + } +} + +async function main() { + const typesVersionSubpaths = Object.keys(packageJson.typesVersions["*"]) + .filter((subpath) => subpath !== "*") + .map((subpath) => `./${subpath}`); + + const exportKeys = packageSubpaths.map(({ exportKey }) => exportKey); + const missingTypesVersions = exportKeys.filter( + (exportKey) => !typesVersionSubpaths.includes(exportKey), + ); + const extraTypesVersions = typesVersionSubpaths.filter( + (subpath) => !exportKeys.includes(subpath), + ); + + assert( + missingTypesVersions.length === 0 && extraTypesVersions.length === 0, + [ + "Package exports and typesVersions drift detected.", + `Missing typesVersions entries for exports: ${formatSubpaths(missingTypesVersions)}`, + `Extra typesVersions entries without exports: ${formatSubpaths(extraTypesVersions)}`, + ].join("\n"), + ); + + for (const { exportKey, specifier } of packageSubpaths) { + assertExportFilesExist(exportKey); + require.resolve(specifier, { paths: [packageRoot] }); + assert( + typesVersionSubpaths.includes(exportKey), + `Missing typesVersions entry for ${exportKey}`, + ); + } + + for (const specifier of loadablePackageSubpaths) { + await loadSpecifier(specifier); + } + + for (const { specifier, missingPeerPattern } of optionalPeerSubpaths) { + await assertOptionalPeerImport(specifier, missingPeerPattern); + } + + for (const specifier of privateSubpaths) { + try { + require.resolve(specifier, { paths: [packageRoot] }); + throw new Error(`Expected ${specifier} to be private`); + } catch (error) { + if (error.code !== "ERR_PACKAGE_PATH_NOT_EXPORTED") { + throw error; + } + } + } + + assertPackedArtifactConsumerInstall(); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/typescript/test/test-core/package-exports.test.ts b/typescript/test/test-core/package-exports.test.ts new file mode 100644 index 0000000000..c3cb8f1db8 --- /dev/null +++ b/typescript/test/test-core/package-exports.test.ts @@ -0,0 +1,74 @@ +import * as fs from "fs"; +import * as path from "path"; + +type PackageJson = { + exports: Record< + string, + { + import: string; + require: string; + types: string; + } + >; + typesVersions: Record>; +}; + +const packageRoot = path.resolve(__dirname, "../.."); +const packageJsonPath = path.join(packageRoot, "package.json"); +const packageJson = JSON.parse( + fs.readFileSync(packageJsonPath, "utf8"), +) as PackageJson; + +const rootTypesVersionKeys = Object.keys(packageJson.typesVersions["*"]).filter( + (subpath) => subpath !== "*", +); + +const subpathExportKeys = Object.keys(packageJson.exports).filter( + (subpath) => subpath !== ".", +); + +describe("package exports", () => { + test("keeps package exports and typesVersions in sync", () => { + expect(subpathExportKeys.map((subpath) => subpath.slice(2)).sort()).toEqual( + rootTypesVersionKeys.sort(), + ); + }); + + test.each(subpathExportKeys)( + "exports %s with matching runtime and type targets", + (subpath) => { + const exportEntry = packageJson.exports[subpath]; + const typesVersionPath = subpath.replace(/^\.\//, ""); + const typesVersionsEntry = + packageJson.typesVersions["*"][typesVersionPath]; + + expect(typesVersionsEntry).toEqual([exportEntry.types.slice(2)]); + for (const field of ["import", "require", "types"] as const) { + expect(typeof exportEntry[field]).toBe("string"); + expect(exportEntry[field].length).toBeGreaterThan(0); + } + }, + ); + + test("exports the documented TypeScript entrypoints", () => { + for (const subpath of [ + "./metrics", + "./models", + "./test-case", + "./evaluate", + ]) { + expect(packageJson.exports[subpath]).toBeDefined(); + } + }); + + test("keeps the legacy camelCase testCase subpath available", () => { + expect(packageJson.exports["./testCase"]).toEqual({ + import: "./dist/test-case/index.js", + require: "./dist/test-case/index.js", + types: "./dist/test-case/index.d.ts", + }); + expect(packageJson.typesVersions["*"].testCase).toEqual([ + "dist/test-case/index.d.ts", + ]); + }); +});