Fix stddev_pop(numeric) and var_pop(numeric), which were incorrectly producing
authorTom Lane <tgl@sss.pgh.pa.us>
Mon, 9 Jul 2007 16:14:06 +0000 (16:14 +0000)
committerTom Lane <tgl@sss.pgh.pa.us>
Mon, 9 Jul 2007 16:14:06 +0000 (16:14 +0000)
the same outputs as stddev_samp() and var_samp() respectively.

src/backend/utils/adt/numeric.c
src/test/regress/expected/aggregates.out

index 1593a0cc36342be2164969733f613d3cb3e685a5..a1abad9b612cc6047cadf1ec380061fe58fe278a 100644 (file)
@@ -2274,7 +2274,10 @@ numeric_stddev_internal(ArrayType *transarray,
        }
        else
        {
-               mul_var(&vN, &vNminus1, &vNminus1, 0);  /* N * (N - 1) */
+               if (sample)
+                       mul_var(&vN, &vNminus1, &vNminus1, 0);  /* N * (N - 1) */
+               else
+                       mul_var(&vN, &vN, &vNminus1, 0);                /* N * N */
                rscale = select_div_scale(&vsumX2, &vNminus1);
                div_var(&vsumX2, &vNminus1, &vsumX, rscale, true);              /* variance */
                if (!variance)
index 34b720f0f433974b234fb876a2421f2ea06cc8e8..74635479e486e8c8c473c2e7513311eee883d617 100644 (file)
@@ -102,7 +102,7 @@ SELECT var_samp(b) FROM aggtest;
 SELECT stddev_pop(b::numeric) FROM aggtest;
     stddev_pop    
 ------------------
- 151.389361431288
+ 131.107032862199
 (1 row)
 
 SELECT stddev_samp(b::numeric) FROM aggtest;
@@ -114,7 +114,7 @@ SELECT stddev_samp(b::numeric) FROM aggtest;
 SELECT var_pop(b::numeric) FROM aggtest;
       var_pop       
 --------------------
- 22918.738754573025
+ 17189.054065929769
 (1 row)
 
 SELECT var_samp(b::numeric) FROM aggtest;