Fix a bug in remquo{,f,l}, in which the quotient didn't always have the correct sign when the remainder was 0. (#317)

Fix a separate bug in remquo alone, in which the remainder and
quotient were both off by a bit in certain cases involving subnormal
remainders.

The bugs affected all platforms except amd64 and i386, on which the
routines are implemented in assembly.

PR:		166463
Submitted by:	Ilya Burylov
MFC after:	2 weeks

Co-authored-by: David Schultz <[email protected]>

Author: ararslan

src/s_remquo.c

@@ -51,7 +51,7 @@ remquo(double x, double y, int *quo)
 		goto fixup;	/* |x|<|y| return x or x-y */
 	    }
 	    if(lx==ly) {
-		*quo = 1;
+		*quo = (sxy ? -1 : 1);
 		return Zero[(u_int32_t)sx>>31];	/* |x|=|y| return x*0*/
 	    }
 	}
@@ -114,6 +114,7 @@ remquo(double x, double y, int *quo)
 
     /* convert back to floating value and restore the sign */
 	if((hx|lx)==0) {			/* return sign(x)*0 */
+	    q &= 0x7fffffff;
 	    *quo = (sxy ? -q : q);
 	    return Zero[(u_int32_t)sx>>31];
 	}
@@ -129,9 +130,9 @@ remquo(double x, double y, int *quo)
 		lx = (lx>>n)|((u_int32_t)hx<<(32-n));
 		hx >>= n;
 	    } else if (n<=31) {
-		lx = (hx<<(32-n))|(lx>>n); hx = sx;
+		lx = (hx<<(32-n))|(lx>>n); hx = 0;
 	    } else {
-		lx = hx>>(n-32); hx = sx;
+		lx = hx>>(n-32); hx = 0;
 	    }
 	}
 fixup:

src/s_remquof.c

@@ -47,7 +47,7 @@ remquof(float x, float y, int *quo)
 	    q = 0;
 	    goto fixup;	/* |x|<|y| return x or x-y */
 	} else if(hx==hy) {
-	    *quo = 1;
+	    *quo = (sxy ? -1 : 1);
 	    return Zero[(u_int32_t)sx>>31];	/* |x|=|y| return x*0*/
 	}
 
@@ -89,6 +89,7 @@ remquof(float x, float y, int *quo)
 
     /* convert back to floating value and restore the sign */
 	if(hx==0) {				/* return sign(x)*0 */
+	    q &= 0x7fffffff;
 	    *quo = (sxy ? -q : q);
 	    return Zero[(u_int32_t)sx>>31];
 	}

src/s_remquol.c

@@ -96,7 +96,7 @@ remquol(long double x, long double y, int *quo)
 		goto fixup;	/* |x|<|y| return x or x-y */
 	    }
 	    if(ux.bits.manh==uy.bits.manh && ux.bits.manl==uy.bits.manl) {
-		*quo = 1;
+		*quo = (sxy ? -1 : 1);
 		return Zero[sx];	/* |x|=|y| return x*0*/
 	    }
 	}
@@ -138,6 +138,7 @@ remquol(long double x, long double y, int *quo)
 
     /* convert back to floating value and restore the sign */
 	if((hx|lx)==0) {			/* return sign(x)*0 */
+	    q &= 0x7fffffff;
 	    *quo = (sxy ? -q : q);
 	    return Zero[sx];
 	}