Revert "Cherry Pick of M1 build #6847 (#6850)"

This reverts commit b7b4c44.

Author: atalman

.github/workflows/build_wheels_macos.yml

@@ -78,7 +78,7 @@ on:
         description: Set to true if setup-miniconda is needed
         required: false
         type: boolean
-        default: true
+        default: false
       delocate-wheel:
         description: "Whether to run delocate-wheel after building."
         required: false