Fix a silly sign bug with multiplier wrapping. no whatsnew.

This commit is contained in:
Jonathan Gevaryahu 2010-04-21 04:30:52 +00:00
parent 410a6f8023
commit f9adad0b2e

View File

@ -946,8 +946,8 @@ static INT16 clip_and_wrap(INT16 cliptemp)
/* clipping & wrapping, just like the patent shows: /* clipping & wrapping, just like the patent shows:
first of all the result should be clamped to 14 bits, between -16384 and 16383 first of all the result should be clamped to 14 bits, between -16384 and 16383
*/ */
while (cliptemp > 16383) cliptemp -= 16384; while (cliptemp > 16383) cliptemp -= 32768;
while (cliptemp < -16384) cliptemp += 16384; while (cliptemp < -16384) cliptemp += 32768;
/* the top 10 bits of this result are visible on the digital output IO pin. /* the top 10 bits of this result are visible on the digital output IO pin.
next, if the top 3 bits of the 14 bit result are all the same, the lowest of those 3 bits plus the next 7 bits are the signed analog output, otherwise the low bits are all forced to match the inverse of the topmost bit, i.e.: next, if the top 3 bits of the 14 bit result are all the same, the lowest of those 3 bits plus the next 7 bits are the signed analog output, otherwise the low bits are all forced to match the inverse of the topmost bit, i.e.:
1x xxxx xxxx xxxx -> 0b10000000 1x xxxx xxxx xxxx -> 0b10000000
@ -977,10 +977,10 @@ static INT16 clip_and_wrap(INT16 cliptemp)
static INT16 matrix_multiply(INT16 a, INT16 b) static INT16 matrix_multiply(INT16 a, INT16 b)
{ {
INT16 result; INT16 result;
while (a>511) { a-=512; } while (a>511) { a-=1024; }
while (a<-512) { a+=512; } while (a<-512) { a+=1024; }
while (b>16383) { b-=16384; } while (b>16383) { b-=32768; }
while (b<-16384) { b+=16384; } while (b<-16384) { b+=32768; }
result = ((a*b)>>9)|1; result = ((a*b)>>9)|1;
#ifdef VERBOSE #ifdef VERBOSE
if (result>16383) fprintf(stderr,"matrix multiplier overflowed! a: %x, b: %x", a, b); if (result>16383) fprintf(stderr,"matrix multiplier overflowed! a: %x, b: %x", a, b);