Merge remote-tracking branch 'origin/develop'

Author: awdeorio

madoop/mapreduce.py

@@ -278,6 +278,13 @@ def group_stage(input_dir, output_dir):
             last_two(inpath), outparent.name, ",".join(outnames),
         )
 
+    # Remove empty output files.  We won't always use the maximum number of
+    # reducers because some MapReduce programs have fewer intermediate keys.
+    for path in sorted(output_dir.iterdir()):
+        if path.stat().st_size == 0:
+            LOGGER.debug("empty partition: rm %s", last_two(path))
+            path.unlink()
+
     # Sort output files
     for path in sorted(output_dir.iterdir()):
         sort_file(path)

setup.py

@@ -14,7 +14,7 @@
     description="A light weight MapReduce framework for education.",
     long_description=LONG_DESCRIPTION,
     long_description_content_type="text/markdown",
-    version="0.4.0",
+    version="0.5.0",
     author="Andrew DeOrio",
     author_email="[email protected]",
     url="https://github.com/eecs485staff/madoop/",

tests/testdata/word_count/correct/grouper-output/part-00001

@@ -0,0 +1,3 @@
+Bye 1
+Hadoop 2
+World 2

tests/testdata/word_count/correct/output/part-00001

@@ -1,3 +1 @@
-Bye 1
-Hadoop 2
-World 2
+Hello 2

tests/testdata/word_count/correct/output/part-00002

@@ -0,0 +1,3 @@
+Bye 1
+Hadoop 2
+World 2

tests/testdata/word_count/correct/output/part-00003

@@ -1,3 +1 @@
-Bye 1
-Hadoop 2
-World 2
+Hello 2