Skip to content

Commit 96318a6

Browse files
ravwojdylaclaude
andcommitted
Remove unused transform modules (evaluation, fasttext, lingoly)
These modules had no imports outside their own tests. Also removes dead render functions from the data browser. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent e219018 commit 96318a6

5 files changed

Lines changed: 0 additions & 430 deletions

File tree

data_browser/src/ExperimentPage.js

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -200,20 +200,6 @@ function renderRaw2JsonStep({step, steps}) {
200200
return {name: "raw2json", description};
201201
}
202202

203-
function renderFastTextTransformStep({step, steps}) {
204-
const description = renderPath({path: step.config.input_path, steps});
205-
return {name: "fasttext2json", description};
206-
}
207-
208-
function renderConvertEvalToDolmaStep({step, steps}) {
209-
const description = <table>
210-
<tbody>
211-
<tr><td>Input:</td><td>{renderPath({path: step.config.input_path, steps})}</td></tr>
212-
</tbody>
213-
</table>;
214-
return {name: "eval2dolma", description};
215-
}
216-
217203
function renderRunClassificationInferenceStep({step, steps}) {
218204
const description = <table>
219205
<tbody>
@@ -517,9 +503,6 @@ const stepRenderers = {
517503
"marin.download.nemotron_cc.download_nemotron_cc.download_nemotron_cc": renderDownloadNemotronCCStep,
518504
"marin.transform.huggingface.dataset_to_eval.hf_dataset_to_jsonl": renderRaw2JsonStep,
519505
"marin.transform.conversation.transform_conversation.transform_hf_dataset": renderRaw2JsonStep,
520-
"marin.transform.fasttext.transform.main": renderFastTextTransformStep,
521-
"marin.transform.evaluation.eval_to_dolma.convert_eval_to_dolma": renderConvertEvalToDolmaStep,
522-
523506
// Inference for data filtering
524507
"marin.processing.classification.inference.run_inference": renderRunClassificationInferenceStep,
525508
"marin.generation.inference.run_inference": renderRunGenerationInferenceStep,

lib/marin/src/marin/transform/evaluation/eval_to_dolma.py

Lines changed: 0 additions & 53 deletions
This file was deleted.

lib/marin/src/marin/transform/fasttext/transform.py

Lines changed: 0 additions & 98 deletions
This file was deleted.

lib/marin/src/marin/transform/lingoly/to_dolma.py

Lines changed: 0 additions & 92 deletions
This file was deleted.

0 commit comments

Comments
 (0)