Skip to content

Commit 2ad1400

Browse files
drernieclaude
drernie
and
claude
committed
Migrate to new Nextflow plugin build system (#323)
Removes the legacy buildSrc/launch.sh/plugins/build.gradle structure and adopts the io.nextflow.nextflow-plugin Gradle plugin (1.0.0-beta.6) per the Seqera migration guide. What changed: - Delete buildSrc/, launch.sh, plugins/build.gradle, plugins/nf-quilt/build.gradle, groovysh helpers, plugins/nf-quilt/src/resources/META-INF/{MANIFEST.MF,extensions.idx} - Move plugin sources from plugins/nf-quilt/src/{main,test,resources} to src/main/groovy, src/test/groovy, src/main/resources (standard layout expected by the new plugin) - Replace settings.gradle with single rootProject.name line - New root build.gradle declaring quiltcore, jackson-databind, jackson-datatype-jsr310, commons-io as runtime deps; nextflowPlugin {} block declares className, provider, extensionPoints (QuiltObserverFactory, QuiltPathFactory, QuiltPathSerializer); inline plugin version 0.9.2 - Rewrite Makefile around assemble / installPlugin / releasePlugin while preserving pkg-test / dyn-test / s3-* targets (now invoke `nextflow` directly instead of launch.sh, install plugin via gradle first) - Update .github/workflows/test.yml artifact paths to build/reports - QuiltSpecification + QuiltPkgTest read MANIFEST.MF from build/tmp/jar/ (where the new gradle plugin generates it during the jar task), with legacy paths kept as fallbacks - Test task gets --add-opens jvmArgs and dependsOn jar so the manifest exists before tests start - Clear gradle.properties (legacy keys no longer needed) Plugin-Version is unchanged (0.9.2). The 1.0.0 release happens in a later PR. Refs #323 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent abfdd28 commit 2ad1400

49 files changed

Lines changed: 137 additions & 877 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,13 @@ jobs:
6060
with:
6161
name: nf-quilt-test-reports-${{ matrix.os }}-${{ matrix.java_version }}
6262
path: |
63-
D:\a\nf-quilt\nf-quilt\plugins\nf-quilt\build\reports\
63+
D:\a\nf-quilt\nf-quilt\build\reports\
6464
overwrite: true
6565
- name: Archive production artifacts (Linux and MacOS)
6666
uses: actions/upload-artifact@v4
6767
if: ${{ always() && matrix.os != 'windows-latest' }}
6868
with:
6969
name: nf-quilt-test-reports-${{ matrix.os }}-${{ matrix.java_version }}
7070
path: |
71-
${{ github.workspace }}/plugins/nf-quilt/build/reports/
71+
${{ github.workspace }}/build/reports/
7272
overwrite: true

‎Makefile‎

Lines changed: 56 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -2,152 +2,112 @@ sinclude .env # create from example.env
22
PROJECT ?= nf-quilt
33
WRITE_BUCKET ?= write-bucket-not-set
44
FRAGMENT ?= &path=.
5-
NF_DIR ?= ../nextflow
6-
NF_GIT ?= $(NF_DIR)/nextflow
7-
NF_BIN ?= ./launch.sh
8-
PID ?= $$$$
95
QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
10-
VERSION ?= $(shell grep 'Plugin-Version' plugins/$(PROJECT)/src/resources/META-INF/MANIFEST.MF | awk '{ print $$2 }')
6+
VERSION ?= $(shell grep "^version" build.gradle | head -1 | awk -F"'" '{ print $$2 }')
117
NXF_VER ?= $(shell cat VERSION)
128
TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=nf-quilt/dest-$(VERSION)$(FRAGMENT)
139
PIPELINE ?= sarek
14-
PIPE_OUT ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
15-
NXF_PLUGINS_TEST_REPOSITORY ?= https://github.com/quiltdata/nf-quilt/releases/download/$(VERSION)/nf-quilt-$(VERSION)-meta.json
10+
PIPE_OUT ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
1611
S3_BASE = s3://$(WRITE_BUCKET)/$(PROJECT)
17-
REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html
12+
REPORT ?= ./build/reports/tests/test/index.html
1813

19-
verify: #compile
14+
.PHONY: all assemble clean test test-all check rebuild install package release verify fast \
15+
check-env coverage pkg-test dyn-test s3-overlay s3-test s3-in s3-out \
16+
pkg-fail path-input deps update refresh
17+
18+
all: assemble
19+
20+
assemble:
21+
./gradlew assemble
22+
23+
clean:
24+
rm -rf .nextflow*
25+
rm -rf work results null
26+
rm -rf build
27+
./gradlew clean
28+
29+
test:
30+
./gradlew test
31+
32+
check:
33+
./gradlew check --warning-mode all
34+
35+
verify:
2036
echo $(WRITE_BUCKET)
2137
./gradlew test ${ONE} || open $(REPORT)
2238

2339
fast:
2440
./gradlew test ${ONE} --fail-fast || open $(REPORT)
2541
# example: make fast ONE="--tests QuiltProductTest"
42+
2643
check-env:
2744
echo $(VERSION)
2845
echo $(WRITE_BUCKET)
2946
echo "$(TEST_URI)"
30-
echo "Use 'make WRITE_BUCKET=<value>' to override"
47+
echo "Use 'make WRITE_BUCKET=<value>' to override"
3148
printenv MAKEFLAGS
3249

33-
clean:
34-
./gradlew clean
35-
rm -rf null results work
36-
rm -rf build */build */*/build plugins/nf-quilt/bin
37-
rm -f .nextflow.log* .launch*classpath
38-
39-
clean-all: clean
40-
rm -rf .gradle buildSrc/.gradle
41-
4250
rebuild:
4351
./gradlew clean build --refresh-dependencies
4452

45-
compile:
46-
./gradlew compileGroovy exportClasspath
47-
@echo "DONE `date`"
48-
49-
nextflow:
50-
if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi
51-
cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd ..
52-
53-
compile-all: nextflow compile
54-
55-
check:
56-
./gradlew check --warning-mode all
57-
58-
.PHONY: clean test test-all all pkg-test tower-test
59-
60-
test: clean compile check verifyCoverage
61-
62-
test-nextflow: clean nextflow-git compile check
63-
64-
test-all: clean compile-all check coverage
53+
test-all: clean test
6554

6655
coverage:
6756
./gradlew jacocoTestReport
68-
open plugins/nf-quilt/build/reports/jacoco/test/html/index.html
57+
open build/reports/jacoco/test/html/index.html || true
6958

70-
verifyCoverage:
71-
./gradlew jacocoTestCoverageVerification
59+
install: assemble
60+
./gradlew installPlugin
7261

73-
groovysh:
74-
./gradlew -q --no-daemon --console=plain --init-script groovysh-task.gradle groovysh
62+
package:
63+
./gradlew packagePlugin
64+
65+
release:
66+
./gradlew releasePlugin
7567

7668
#
77-
# Create packages
69+
# Create packages (real-S3 integration tests).
70+
# These run the plugin via `nextflow` directly. Requires `nextflow` on PATH.
7871
#
7972

80-
pkg-test: compile #-all
73+
pkg-test: install
8174
echo "$(TEST_URI)"
82-
$(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
75+
nextflow run ./main.nf -profile standard -plugins $(PROJECT)@$(VERSION) --outdir "$(TEST_URI)"
8376

84-
dyn-test: compile #-all
85-
$(NF_BIN) run wf/main.dynamic.nf -profile standard -plugins $(PROJECT)
77+
dyn-test: install
78+
nextflow run wf/main.dynamic.nf -profile standard -plugins $(PROJECT)@$(VERSION)
8679

87-
s3-overlay: compile
88-
$(NF_BIN) run ./main.nf --plugins $(PROJECT) --outdir "$(S3_BASE)/s3-overlay" --input "$(S3_BASE)/s3-in"
80+
s3-overlay: install
81+
nextflow run ./main.nf --plugins $(PROJECT)@$(VERSION) --outdir "$(S3_BASE)/s3-overlay" --input "$(S3_BASE)/s3-in"
8982

90-
s3-test: compile
91-
$(NF_BIN) run ./main.nf --outdir "$(S3_BASE)/s3-test" --input "$(S3_BASE)/s3-in"
83+
s3-test: install
84+
nextflow run ./main.nf --outdir "$(S3_BASE)/s3-test" --input "$(S3_BASE)/s3-in"
9285

93-
s3-in: compile
94-
$(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" --input "$(S3_BASE)/s3-in"
86+
s3-in: install
87+
nextflow run ./main.nf -profile standard -plugins $(PROJECT)@$(VERSION) --outdir "$(TEST_URI)" --input "$(S3_BASE)/s3-in"
9588

96-
s3-out: compile
97-
$(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(S3_BASE)/s3-out"
89+
s3-out: install
90+
nextflow run ./main.nf -profile standard -plugins $(PROJECT)@$(VERSION) --outdir "$(S3_BASE)/s3-out"
9891

99-
pkg-fail: compile
92+
pkg-fail: install
10093
echo "$(TEST_URI)"
101-
$(NF_BIN) run wf/fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
94+
nextflow run wf/fail.nf -profile standard -plugins $(PROJECT)@$(VERSION) --outdir "$(TEST_URI)"
10295

103-
path-input: compile
96+
path-input: install
10497
mkdir -p work
10598
date > work/COPY_THIS.md
10699
echo "$(TEST_URI)"
107-
$(NF_BIN) run wf/main.path.nf -profile standard -plugins $(PROJECT) --outdir "./results"
108-
109-
tower-test: $(NF_BIN)
110-
$(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein -with-tower -r main -latest --pub "$(TEST_URI)"
111-
112-
#
113-
# Production Testing
114-
#
115-
116-
nf-git-ver: $(NF_GIT)
117-
NXF_VER=$(NXF_VER) $(NF_GIT) -v
118-
119-
120-
$(PIPELINE): nf-git-ver
121-
NXF_PLUGINS_TEST_REPOSITORY=$(NXF_PLUGINS_TEST_REPOSITORY) NXF_VER=$(NXF_VER) $(NF_GIT) run nf-core/$(PIPELINE) -r master -profile test,docker -plugins $(PROJECT)@$(VERSION) --outdir "$(PIPE_OUT)"
122-
123-
fetchngs: nf-git-ver
124-
NXF_PLUGINS_TEST_REPOSITORY=$(NXF_PLUGINS_TEST_REPOSITORY) NXF_VER=$(NXF_VER) $(NF_GIT) run nf-core/fetchngs -r master -profile test,docker -plugins $(PROJECT)@$(VERSION) --input ../nf-quilt/wf/ids.csv --outdir s3://$(WRITE_BUCKET)/nf-quilt/fetchngs
100+
nextflow run wf/main.path.nf -profile standard -plugins $(PROJECT)@$(VERSION) --outdir "./results"
125101

126102
#
127103
# Show dependencies
128104
#
129105

130106
deps:
131-
./gradlew -q ${mm}dependencies
107+
./gradlew -q dependencies
132108

133109
update:
134-
./gradlew useLatestVersions
135-
make check
110+
./gradlew dependencyUpdates
136111

137112
refresh:
138113
./gradlew --refresh-dependencies dependencies
139-
140-
install: compile
141-
./gradlew copyPluginZip
142-
rm -rf ${HOME}/.nextflow/plugins/$(PROJECT)-${VERSION}
143-
cp -r build/plugins/$(PROJECT)-${VERSION} ${HOME}/.nextflow/plugins/
144-
145-
#
146-
# Upload JAR artifacts to Maven Central
147-
#
148-
149-
publish:
150-
echo "Ensure you have set 'github_organization=<owner>' in ~/.gradle/gradle.properties"
151-
ls $(HOME)/.gradle/gradle.properties # create locally or globally if it does not exist
152-
./gradlew :plugins:$(PROJECT):upload
153-
./gradlew :plugins:publishIndex

0 commit comments

Comments
 (0)