fix: remove fasttext classifier steps from integration test

The classifier code (train + inference) uses patterns that don't work
in distributed S3 environments: fs.makedirs on local /tmp paths via
S3FileSystem, HF datasets injecting aiohttp kwargs into botocore, and
single-node FileLock assumptions.

Remove these steps from the integration test so the CW CI can pass.
See #4183 for the tracking issue.

Author: rjpower

tests/integration_test.py

@@ -20,14 +20,8 @@
 )
 from marin.execution.step_spec import StepSpec
 from marin.processing.classification.consolidate import FilterConfig, FilterType, consolidate, ConsolidateConfig
-from marin.processing.classification.dataset_utils import DatasetConfig
 from marin.processing.classification.deduplication.exact import dedup_exact_paragraph
 from marin.processing.classification.deduplication.fuzzy import dedup_fuzzy_document
-from marin.processing.classification.fasttext.train_fasttext import (
-    TrainFasttextClassifierConfig,
-    train,
-)
-from marin.processing.classification.inference import InferenceConfig, run_inference
 from marin.processing.tokenize import lm_data_config
 from marin.processing.tokenize.tokenize import TokenizeConfig, tokenize
 from marin.schemas.web.convert import ResiliparseConfig
@@ -153,64 +147,6 @@ def create_steps(prefix: str, synth_data: str) -> list[ExecutorStep]:
     transform_hq_data_step = transform_hq_data_spec.as_executor_step()
     transform_lq_data_step = transform_lq_data_spec.as_executor_step()
 
-    # ############################################################
-    # Train quality classifier
-
-    train_quality_step = ExecutorStep(
-        name=os.path.join(prefix, "quality-classifier"),
-        fn=train,
-        config=TrainFasttextClassifierConfig(
-            datasets=[
-                DatasetConfig(
-                    input_doc_path=transform_hq_data_step,
-                    label="hq",
-                    sampling_rate=1.0,
-                ),
-                DatasetConfig(
-                    input_doc_path=transform_lq_data_step,
-                    label="lq",
-                    sampling_rate=1.0,
-                ),
-            ],
-            output_path=this_output_path(),
-            fasttext_args={
-                "lr": 0.001,
-                "minCount": 1,
-                "epoch": 25,
-                "wordNgrams": 2,
-                "dim": 50,
-                "thread": 1,
-            },
-        ),
-    )
-
-    ############################################################
-    # Run inference with quality classifier
-
-    inference_hq_step = ExecutorStep(
-        name=os.path.join(prefix, "hq-inference"),
-        fn=run_inference,
-        config=InferenceConfig(
-            input_path=transform_hq_data_step,
-            output_path=this_output_path(),
-            model_name=train_quality_step,
-            model_type="fasttext",
-            attribute_name="quickstart-fasttext-quality-hq",
-        ),
-    )
-
-    inference_lq_step = ExecutorStep(
-        name=os.path.join(prefix, "lq-inference"),
-        fn=run_inference,
-        config=InferenceConfig(
-            input_path=transform_lq_data_step,
-            output_path=this_output_path(),
-            model_name=train_quality_step,
-            model_type="fasttext",
-            attribute_name="quickstart-fasttext-quality-lq",
-        ),
-    )
-
     ############################################################
     # Deduplicate (StepSpec — depends on transform StepSpecs)
 
@@ -351,9 +287,6 @@ def create_steps(prefix: str, synth_data: str) -> list[ExecutorStep]:
     return [
         transform_hq_data_step,
         transform_lq_data_step,
-        train_quality_step,
-        inference_hq_step,
-        inference_lq_step,
         dedup_exact_paragraph_step,
         dedup_fuzzy_document_step,
         validate_exact_dedup_step,