Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: internal
packages:
- "@azure-tools/typespec-java"
---

Unify the emitter and its e2e tests into a single npm project: `emitter-tests` is no longer a standalone npm project consuming a packed `.tgz`, but part of the `@azure-tools/typespec-java` workspace package. The e2e tests now run against the workspace build via `pnpm run regenerate` / `pnpm run test:java:e2e`, resolving the locally built emitter by package name through Node self-reference (the `emitter-tests` folder has no `package.json`, so `emit: ["@azure-tools/typespec-java"]` in its `tspconfig.yaml` resolves to the parent package).
25 changes: 11 additions & 14 deletions .github/workflows/ci-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,23 +48,20 @@ jobs:
- name: Install dependencies
run: pnpm install

# The emitter compiles against these peer packages; Generate.ps1 -> Setup.ps1
# -> Build-TypeSpec.ps1 runs `pnpm build` for the emitter, which needs them
# already built.
# The emitter compiles against these peer packages; `pnpm run regenerate`
# (Generate.ps1) runs `pnpm build` for the emitter, which needs them already
# built.
- name: Build dependencies
run: pnpm turbo run --filter "@azure-tools/typespec-java^..." build

# emitter-tests is a standalone npm+Maven project (not in the pnpm
# workspace). Generate.ps1 calls Setup.ps1 (builds the emitter + Azure jar,
# npm install) and regenerates the SDK from the spector specs into the
# gitignored src/main/java. Spector-Tests.ps1 then runs the JUnit tests
# against the Spector mock server.
# The e2e tests live in the same @azure-tools/typespec-java package (Maven+tsp
# assets under emitter-tests/). `pnpm run regenerate` builds the emitter and
# regenerates the SDK from the spector specs into the gitignored src/main/java;
# `pnpm run test:java:e2e` runs the JUnit tests against the Spector mock server.
- name: Regenerate
shell: pwsh
run: ./Generate.ps1
working-directory: packages/typespec-java/emitter-tests
run: pnpm run regenerate
working-directory: packages/typespec-java

- name: Spector tests
shell: pwsh
run: ./Spector-Tests.ps1
working-directory: packages/typespec-java/emitter-tests
run: pnpm run test:java:e2e
working-directory: packages/typespec-java
2 changes: 1 addition & 1 deletion .github/workflows/nightly-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
- name: Java
lang: java
package-filter: "@azure-tools/typespec-java"
working-directory: packages/typespec-java/emitter-tests
working-directory: packages/typespec-java
mise-install-args: "java maven"
run-prepare: false
run-regenerate: true
Expand Down
5 changes: 1 addition & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -241,18 +241,15 @@ packages/typespec-java/src/**
packages/typespec-java/test/**
# Java emitter.jar + assets staged from the core generator build.
packages/typespec-java/generator/**
# Packed tarball produced by Build-TypeSpec.ps1 (consumed by emitter-tests).
packages/typespec-java/*.tgz

# java emitter-tests (standalone Maven+npm spector suite)
# java emitter-tests (Maven+tsp spector suite, part of the typespec-java package)
# Generated SDK + generated tests are regenerated, never committed; only
# hand-written tests, specs, customization and the harness are tracked.
packages/typespec-java/emitter-tests/src/main/**
packages/typespec-java/emitter-tests/**/generated/**
packages/typespec-java/emitter-tests/tsp-output/**
packages/typespec-java/emitter-tests/specs/**
packages/typespec-java/emitter-tests/node_modules/**
packages/typespec-java/emitter-tests/package-lock.json
packages/typespec-java/emitter-tests/tsp-spector-coverage-*.json

# Generated linter rule reference pages (produced by `tspd doc` / regen-docs via
Expand Down
7 changes: 3 additions & 4 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,9 @@ packages/typespec-ts/test/azure-integration/generated/
packages/typespec-ts/test/modular-integration/generated/
packages/typespec-ts/test/azure-modular-integration/generated/

# typespec-java emitter-tests is a standalone npm project (excluded from the pnpm
# workspace) with its own .prettierrc.yaml that loads the typespec prettier plugin
# from its own node_modules. Those aren't installed in the root format job, so root
# prettier must not descend into it; the project formats itself via its own scripts.
# typespec-java emitter-tests holds Maven+tsp e2e assets whose .tsp specs and
# hand-written sources are synced verbatim from core (see SyncTests.ps1); root
# prettier must not reformat these upstream-sourced files.
packages/typespec-java/emitter-tests/
# Emitter TypeScript sources (except options.ts) are copied from core at build time.
packages/typespec-java/src/**
Expand Down
22 changes: 11 additions & 11 deletions eng/pipelines/jobs/spector-coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -112,23 +112,23 @@ jobs:
- script: pnpm run build
displayName: Build (turbo cache restore)

# emitter-tests is a standalone npm+Maven project (not in the pnpm
# workspace). The turbo cache only restores dist/** outputs, not the
# Maven emitter.jar, so run Generate.ps1 WITHOUT -SkipBuild: it builds the
# emitter jar, then regenerates the SDK from the spector specs.
# Spector-Tests.ps1 starts the mock server, runs the JUnit tests and
# produces tsp-spector-coverage-java.json.
- pwsh: ./Generate.ps1
# The e2e tests live in the @azure-tools/typespec-java package (Maven+tsp
# assets under emitter-tests/). The turbo cache only restores dist/** outputs,
# not the Maven emitter.jar, so `pnpm run regenerate` builds the emitter jar,
# then regenerates the SDK from the spector specs. `pnpm run test:java:e2e`
# starts the mock server, runs the JUnit tests and produces
# tsp-spector-coverage-java.json.
- pwsh: pnpm run regenerate
displayName: "Regenerate Java test code"
workingDirectory: packages/typespec-java/emitter-tests
workingDirectory: packages/typespec-java

- pwsh: ./Spector-Tests.ps1
- pwsh: pnpm run test:java:e2e
displayName: "Run Java spector tests"
workingDirectory: packages/typespec-java/emitter-tests
workingDirectory: packages/typespec-java

- template: /eng/pipelines/templates/upload-spector-coverage.yml
parameters:
PackagePath: packages/typespec-java/emitter-tests
PackagePath: packages/typespec-java
GeneratorName: "@azure-tools/typespec-java"

- job: spector_ts
Expand Down
27 changes: 0 additions & 27 deletions packages/typespec-java/Build-TypeSpec.ps1

This file was deleted.

49 changes: 39 additions & 10 deletions packages/typespec-java/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,9 @@ refresh `core.patch` if its context no longer applies.
# From the repo root, install workspace dependencies.
pnpm install

# From the repo root, build the dependencies first. The ^... filter builds
# @azure-tools/typespec-java dependencies without building typespec-java itself.
pnpm turbo run --filter "@azure-tools/typespec-java^..." build

# Then build and pack @azure-tools/typespec-java. This builds the generator
# (emitter.jar via Maven + core.patch, requires JDK 11+ and Maven), builds the
# emitter TypeScript, and packs the .tgz consumed by emitter-tests.
cd packages/typespec-java
pwsh ./Build-TypeSpec.ps1
# From the repo root, build typespec-java along with all its dependencies.
# Use run-all so pnpm does not auto-install concurrently during the Turbo build.
pnpm run-all --filter "@azure-tools/typespec-java..." build
```

### Pinning the core commit (`core-commit.json`)
Expand All @@ -76,12 +70,47 @@ force pnpm to refresh the local install state and rerun the command:

```powershell
pnpm install --force
pnpm turbo run --filter "@azure-tools/typespec-java^..." build
pnpm run-all --filter "@azure-tools/typespec-java..." build
```

Changing the npm registry has been observed to clear this symptom, possibly because
pnpm re-resolves packages or relinks local binaries after the registry setting changes.

## Debugging

### Debugging TypeScript code

Build the package first, then run the TypeSpec compiler under the Node.js debugger from
`packages/typespec-java`:

```shell
node --inspect-brk node_modules/@typespec/compiler/dist/src/core/cli/cli.js compile emitter-tests/<tsp-file>
```

Attach a debugger to port 9229 and set breakpoints in `src/emitter.ts`,
`src/code-model-builder.ts`, or their compiled counterparts under `dist/src`.

### Debugging Java code

TypeScript passes the code model and emitter options to Java through the generated
`emitter-tests/tsp-output/code-model.yaml` file. To debug the Java generator directly:

1. Build the package so `Copy-Sources.ps1` creates the patched generator copy under `generator/`.
2. Update `DEFAULT_OUTPUT_DIR` in
`generator/http-client-generator/src/main/java/com/microsoft/typespec/http/client/generator/Main.java`
to the directory containing the `code-model.yaml` to debug.
3. Run `com.microsoft.typespec.http.client.generator.Main.main()` from an IDE with these VM options:

```text
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
```

The copied generator is recreated on each build, so do not commit changes under `generator/`.
Emitter options used by `Main` are defined in
`generator/http-client-generator/src/main/java/com/microsoft/typespec/http/client/generator/model/EmitterOptions.java`.
When debugging this way, temporarily align them with the options in the relevant `tspconfig.yaml`;
for example, set `flavor` to `azure`.

## Before making a Pull request

Make sure to run the following commands:
Expand Down
6 changes: 3 additions & 3 deletions packages/typespec-java/Copy-Sources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ try {
# http-client-generator-mgmt and http-client-generator-core. The two test modules
# below live in a separate (never-activated) Maven `test` profile and hold ~38MB
# of generated Java test sources that are not part of the emitter build, so skip
# copying them -- it is the bulk of the copy time. (The Azure emitter-tests
# project syncs http-client-generator-test straight from core via SyncTests.ps1,
# not from this copy.)
# copying them -- it is the bulk of the copy time. (This package's e2e tests
# under emitter-tests/ sync http-client-generator-test straight from core via
# SyncTests.ps1, not from this copy.)
$excludedGeneratorModules = @("http-client-generator-test", "http-client-generator-clientcore-test")

Write-Host "Copy generator sources from core"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
# Before running this script the 'tsp' profile must be built, 'mvn install -P local,tsp'.
param (
[int] $Parallelization = [Environment]::ProcessorCount,
# skip the emitter build in Setup.ps1 (only pack + install); use when the
# package was already built by a prior step (e.g. the repo-wide `pnpm build`
# in CI) to avoid a redundant second build
# skip the emitter build; use when the package was already built by a prior step
# (e.g. the repo-wide `pnpm build` in CI) to avoid a redundant second build
[switch] $SkipBuild = $false
)

Expand Down Expand Up @@ -122,6 +121,9 @@ $generateScript = {
}

$tspTrace = "--trace import-resolution --trace projection --trace typespec-java"
# The emitter is resolved by package name from emitter-tests/tspconfig.yaml via Node
# self-reference (this folder has no package.json, so the nearest one is the parent
# typespec-java package). No --emit needed.
$tspCommand = "npx --no-install tsp compile $tspFile $tspOptions $tspTrace"

# output of "tsp compile" seems trigger powershell error or exit, hence the "2>&1"
Expand Down Expand Up @@ -153,9 +155,25 @@ $generateScript = {
}
}

Push-Location $PSScriptRoot
# This script lives at the typespec-java package root; the Maven/tsp e2e assets it
# operates on are under ./emitter-tests, so it runs with the working directory pushed
# into that folder (all ./tsp, ./src, ./specs paths are relative to it).
Push-Location (Join-Path $PSScriptRoot "emitter-tests")
try {
./Setup.ps1 -SkipBuild:$SkipBuild
if (-not $SkipBuild) {
# Build the emitter (emitter.jar via Build-Generator.ps1 + tsc) from the
# workspace package. The e2e tests consume it directly from the built dist +
# generator, resolved by package name via tspconfig.yaml -- no .tgz.
Push-Location ..
try {
pnpm build
if ($LASTEXITCODE -ne 0) {
throw "Failed to build @azure-tools/typespec-java"
}
} finally {
Pop-Location
}
}

New-Item -Path ./existingcode/src/main/java/tsptest/ -ItemType Directory -Force | Out-Null

Expand Down Expand Up @@ -187,9 +205,10 @@ try {
Copy-Item -Path ./existingcode/src/main/java/tsptest/partialupdate -Destination ./src/main/java/tsptest/partialupdate -Recurse -Force
Remove-Item ./existingcode -Recurse -Force

# generate for http-specs/azure-http-specs test sources
Copy-Item -Path node_modules/@typespec/http-specs/specs -Destination ./ -Recurse -Force
Copy-Item -Path node_modules/@azure-tools/azure-http-specs/specs -Destination ./ -Recurse -Force
# generate for http-specs/azure-http-specs test sources (installed under the
# typespec-java package's node_modules by the workspace `pnpm install`).
Copy-Item -Path ../node_modules/@typespec/http-specs/specs -Destination ./ -Recurse -Force
Copy-Item -Path ../node_modules/@azure-tools/azure-http-specs/specs -Destination ./ -Recurse -Force

$specFiles = Get-ChildItem ./specs -Include "main.tsp","old.tsp" -File -Recurse
# ensure multi-service client specs are processed even though they do not match the default filter
Expand Down
47 changes: 47 additions & 0 deletions packages/typespec-java/Spector-Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#Requires -Version 7.0

# Run the Spector tests for @azure-tools/typespec-java.
#
# Prerequisite: the SDK has been regenerated via Generate.ps1 (which builds the
# emitter and regenerates emitter-tests/src/main/java from the spector specs). This
# starts the Spector mock server, compiles and runs the JUnit tests against the
# generated SDK, then stops the server.

$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 3.0

# This script lives at the typespec-java package root, so the pnpm spector-* scripts
# (and node_modules) resolve from here directly. The Maven project (pom.xml, generated
# sources, coverage file) is under ./emitter-tests, entered only for `mvn` below.
Set-Location $PSScriptRoot

Write-Host "Starting the Spector server"
pnpm run spector-start

try {
Write-Host "Compile and run the tests"
Push-Location (Join-Path $PSScriptRoot "emitter-tests")
try {
mvn clean test --no-transfer-progress -T 1C
if ($LASTEXITCODE -ne 0) {
throw "Spector tests failed"
}
}
finally {
Pop-Location
}
}
finally {
Write-Host "Stopping the Spector server"
pnpm run spector-stop

# Stage the coverage report where the ADO upload-spector-coverage.yml template
# expects it: <package>/node_modules/@azure-tools/azure-http-specs/spec-coverage.json.
$coverageSource = "./emitter-tests/tsp-spector-coverage-java.json"
$coverageDest = "./node_modules/@azure-tools/azure-http-specs/spec-coverage.json"
if (Test-Path $coverageSource) {
Copy-Item $coverageSource $coverageDest -Force
}
}

Write-Host "Finished running the Spector tests"
Loading
Loading