Make dupekit a pure Rust workspace (#2170)

Author: yonromai

docker/marin/Dockerfile.cluster

@@ -11,7 +11,7 @@ RUN /tmp/google-cloud-sdk/install.sh --quiet --path-update false --command-compl
 RUN /tmp/google-cloud-sdk/bin/gcloud components install alpha --quiet
 
 # Rust toolchain
-FROM rust:1.82-slim AS rust-builder
+FROM rust:1.91-slim AS rust-builder
 
 
 # Main build

docker/marin/Dockerfile.vllm

@@ -11,7 +11,7 @@ RUN /tmp/google-cloud-sdk/install.sh --quiet --path-update false --command-compl
 RUN /tmp/google-cloud-sdk/bin/gcloud components install alpha --quiet
 
 # Rust toolchain
-FROM rust:1.82-slim AS rust-builder
+FROM rust:1.91-slim AS rust-builder
 
 
 # Main build

lib/dupekit/README.md

@@ -1,6 +1,16 @@
 dupekit
 ---
-🚧 WIP 🚧
+Raison d'être: Home for the Rust code used for text deduplication.
+
+## Install
+
+- **Locally:** This code is auto-magically built by `uv` via Cargo and [Maturin](https://github.com/PyO3/maturin). You
+might need to install them (e.g., `brew install maturin rust` on macOS).
+- **Cluster:** This code is compiled as part of the Docker build (`uv pip install -e ...` step): Maturin builds the Rust
+code and places it in the system `site-packages` (e.g., `/home/ray/anaconda3/lib/python3.11/site-packages/dupekit/dupekit.abi3.so`).
+
+**Note:** What about making `dupekit` a hybrid Python/Rust Maturin workspace? We tried and experienced issues getting
+the Docker build to work while keeping it simple—a simple Rust workspace helps keep the setup clean.
 
 ## Benchmarking
 

lib/dupekit/dupekit/dupekit.pyi lib/dupekit/dupekit.pyilib/dupekit/dupekit/dupekit.pyi renamed to lib/dupekit/dupekit.pyi

@@ -11,6 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-# TODO (rav): remove F403 when we stabilize dupekit
-from .dupekit import *  # noqa: F403

lib/dupekit/dupekit/__init__.py …in/processing/classification/__init__.pylib/dupekit/dupekit/__init__.py renamed to lib/marin/src/marin/processing/classification/__init__.py lib/dupekit/dupekit/__init__.py renamed to lib/marin/src/marin/processing/classification/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 The Marin Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

lib/marin/src/marin/processing/classification/deduplication/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from dupekit.text_cleaning import clean_text
+from marin.processing.classification.deduplication.text_cleaning import clean_text
 
 
 def test_clean_text():