vircam_stats.c

00001 /* $Id: vircam_stats.c,v 1.20 2009/02/23 10:45:50 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2009/02/23 10:45:50 $
00024  * $Revision: 1.20 $
00025  * $Name: vcam-1_3_2 $
00026  */
00027 
00028 /* Includes */
00029 
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033 
00034 #include <math.h>
00035 #include <string.h>
00036 #include <cpl.h>
00037 #include <cxtypes.h>
00038 
00039 #include "vircam_stats.h"
00040 #include "vircam_utils.h"
00041 
00042 /* Static subroutine prototypes */
00043 
00044 static float kselect(float *a, int n, int k);
00045 static double dkselect(double *a, int n, int k);
00046 static float histexam(int *histo, int nhist, int level);
00047 
00061 /*---------------------------------------------------------------------------*/
00087 /*---------------------------------------------------------------------------*/
00088 
00089 extern float vircam_med(float *data, unsigned char *bpm, long npts) {
00090     int i,j,is_even,ilevel;
00091     float *buf,value;
00092 
00093     /* Is there any point being here? */
00094 
00095     if (npts == 0)
00096         return(CX_MAXFLOAT);
00097 
00098     /* If there is not BPM, then just do a straight forward median */
00099 
00100     buf = cpl_malloc(npts*sizeof(*buf));
00101     if (bpm == NULL) {
00102         is_even = !(npts & 1);
00103         memmove((char *)buf,(char *)data,npts*sizeof(float));
00104         if (is_even) {
00105             ilevel = npts/2 - 1;
00106             value = kselect(buf,npts,ilevel);
00107             ilevel = npts/2;
00108             value = 0.5*(value + kselect(buf,npts,ilevel));
00109         } else {
00110             ilevel = npts/2;
00111             value = kselect(buf,npts,ilevel);
00112         }
00113 
00114     /* Otherwise get rid of the dodgy values and then do the median */
00115 
00116     } else {
00117         j = 0;
00118         for (i = 0; i < npts; i++) {
00119             if (bpm[i] == 0)
00120                 buf[j++] = data[i];
00121         }
00122         if (j == 0) {
00123             cpl_free(buf);
00124             value = CX_MAXFLOAT;
00125             return(value);
00126         }
00127         is_even = !(j & 1);
00128         if (is_even) {
00129             ilevel = j/2 - 1;
00130             value = kselect(buf,j,ilevel);
00131             ilevel = j/2;
00132             value = 0.5*(value + kselect(buf,j,ilevel));
00133         } else {
00134             ilevel = j/2;
00135             value = kselect(buf,j,ilevel);
00136         }
00137     }
00138     cpl_free(buf);
00139     return(value);
00140 }
00141 
00142 /*---------------------------------------------------------------------------*/
00168 /*---------------------------------------------------------------------------*/
00169 
00170 extern double vircam_dmed(double *data, unsigned char *bpm, long npts) {
00171     int i,j,is_even,ilevel;
00172     double *buf,value;
00173 
00174     /* If there is not BPM, then just do a straight forward median */
00175 
00176     buf = cpl_malloc(npts*sizeof(*buf));
00177     if (bpm == NULL) {
00178         is_even = !(npts & 1);
00179         memmove((char *)buf,(char *)data,npts*sizeof(double));
00180         if (is_even) {
00181             ilevel = npts/2 - 1;
00182             value = dkselect(buf,npts,ilevel);
00183             ilevel = npts/2;
00184             value = 0.5*(value + dkselect(buf,npts,ilevel));
00185         } else {
00186             ilevel = npts/2;
00187             value = dkselect(buf,npts,ilevel);
00188         }
00189 
00190     /* Otherwise get rid of the dodgy values and then do the median */
00191 
00192     } else {
00193         j = 0;
00194         for (i = 0; i < npts; i++) {
00195             if (bpm[i] == 0)
00196                 buf[j++] = data[i];
00197         }
00198         if (j == 0) {
00199             cpl_free(buf);
00200             value = CX_MAXDOUBLE;
00201             return(value);
00202         }
00203         is_even = !(j & 1);
00204         if (is_even) {
00205             ilevel = j/2 - 1;
00206             value = dkselect(buf,j,ilevel);
00207             ilevel = j/2;
00208             value = 0.5*(value + dkselect(buf,j,ilevel));
00209         } else {
00210             ilevel = j/2;
00211             value = dkselect(buf,j,ilevel);
00212         }
00213     }
00214     cpl_free(buf);
00215     return(value);
00216 }
00217 
00218 /*---------------------------------------------------------------------------*/
00244 /*---------------------------------------------------------------------------*/
00245 
00246 extern float vircam_mean(float *data, unsigned char *bpm, long npts) {
00247     int i,n;
00248     float sum,value;
00249 
00250     /* Separate sections depending on whether there is a BPM or not */
00251 
00252     sum = 0.0;
00253     if (bpm == NULL) {
00254         n = npts;
00255         for (i = 0; i < npts; i++) 
00256             sum += data[i];
00257     } else {
00258         n = 0;
00259         for (i = 0; i < npts; i++) {
00260             if (bpm[i] == 0) {
00261                 sum += data[i];
00262                 n++;
00263             }
00264         }
00265     }
00266     if (n > 0)
00267         value = sum/(float)n;
00268     else
00269         value = CX_MAXFLOAT;
00270     return(value);
00271 }
00272 
00273 /*---------------------------------------------------------------------------*/
00299 /*---------------------------------------------------------------------------*/
00300 
00301 extern double vircam_dmean(double *data, unsigned char *bpm, long npts) {
00302     int i,n;
00303     double sum,value;
00304 
00305     /* Separate sections depending on whether there is a BPM or not */
00306 
00307     sum = 0.0;
00308     if (bpm == NULL) {
00309         n = npts;
00310         for (i = 0; i < npts; i++) 
00311             sum += data[i];
00312     } else {
00313         n = 0;
00314         for (i = 0; i < npts; i++) {
00315             if (bpm[i] == 0) {
00316                 sum += data[i];
00317                 n++;
00318             }
00319         }
00320     }
00321     if (n > 0)
00322         value = sum/(float)n;
00323     else
00324         value = CX_MAXDOUBLE;
00325     return(value);
00326 }
00327 
00328 /*---------------------------------------------------------------------------*/
00360 /*---------------------------------------------------------------------------*/
00361 
00362 extern int vircam_meansig(float *data, unsigned char *bpm, long npts, 
00363                           float *mean, float *sig) {
00364     int i,n;
00365     double sum,sum2,d;
00366     const char *fctid = "vircam_meansig";
00367 
00368     /* Separate sections depending on whether there is a BPM or not */
00369 
00370     sum = 0.0;
00371     sum2 = 0.0;
00372     if (bpm == NULL) {
00373         n = npts;
00374         for (i = 0; i < npts; i++) {
00375             d = (double)(data[i]);
00376             sum += d;
00377             sum2 += d*d;
00378         }
00379     } else {
00380         n = 0;
00381         for (i = 0; i < npts; i++) {
00382             if (bpm[i] == 0) {
00383                 d = (double)(data[i]);
00384                 sum += d;
00385                 sum2 += d*d;
00386                 n++;
00387             }
00388         }
00389     }
00390 
00391     /* Check whether we can do the mean and sigma calculations */
00392 
00393     switch (n) {
00394     case 0:
00395         *mean = CX_MAXFLOAT;
00396         *sig = CX_MAXFLOAT;
00397         cpl_msg_warning(fctid,"All values flagged as bad\n");
00398         return(VIR_WARN);
00399     case 1:
00400         *mean = (float)sum;
00401         *sig = 0.0;
00402         return(VIR_OK);
00403     default:
00404         sum /= (double)n;
00405         *mean = (float)sum;
00406         sum2 = sum2/(double)n - sum*sum;
00407         *sig = (float)sqrt(max(1.0e-12,sum2));
00408         return(VIR_OK);
00409     }
00410 }
00411 
00412 /*---------------------------------------------------------------------------*/
00449 /*---------------------------------------------------------------------------*/
00450 
00451 extern int vircam_meansigcut(float *data, unsigned char *bpm, long npts,
00452                              float lcut, float hcut, float *mean, float *sig) {
00453     int i,n;
00454     double sum,sum2;
00455     const char *fctid = "vircam_meansigcut";
00456 
00457     /* Separate sections depending on whether there is a BPM or not */
00458 
00459     sum = 0.0;
00460     sum2 = 0.0;
00461     if (bpm == NULL) {
00462         n = 0;
00463         for (i = 0; i < npts; i++) {
00464             if (data[i] > lcut && data[i] < hcut) {
00465                 sum += data[i];
00466                 sum2 += data[i]*data[i];
00467                 n++;
00468             }
00469         }
00470     } else {
00471         n = 0;
00472         for (i = 0; i < npts; i++) {
00473             if (bpm[i] == 0 && data[i] > lcut && data[i] < hcut) {
00474                 sum += data[i];
00475                 sum2 += data[i]*data[i];
00476                 n++;
00477             }
00478         }
00479     }
00480 
00481     /* Check whether we can do the mean and sigma calculations */
00482 
00483     switch (n) {
00484     case 0:
00485         *mean = CX_MAXFLOAT;
00486         *sig = CX_MAXFLOAT;
00487         cpl_msg_warning(fctid,"All values flagged as bad\n");
00488         return(VIR_WARN);
00489     case 1:
00490         *mean = (float)sum;
00491         *sig = 0.0;
00492         return(VIR_OK);
00493     default:
00494         sum /= (double)n;
00495         *mean = (float)sum;
00496         sum2 = sum2/(double)n - sum*sum;
00497         *sig = (float)sqrt(max(1.0e-12,sum2));
00498         return(VIR_OK);
00499     }
00500 }
00501 
00502 /*---------------------------------------------------------------------------*/
00540 /*---------------------------------------------------------------------------*/
00541 
00542 extern void vircam_qmedsig(float *data, unsigned char *bpm, long npts,
00543                            float thresh, int niter, float lowv, float highv,
00544                            float *median, float *sigma) {
00545     int *histo,nbins,nhist,ilev,iclip,nhist2,halflev,quartlev;
00546     int irej,jst,j;
00547     long i;
00548     float mlev,qlev;
00549     unsigned char *b;
00550 
00551     /* Right, first thing is to histogram the data.  Cut values below
00552        and above the 'ceiling' values */
00553 
00554     if (bpm == NULL) 
00555         b = cpl_calloc(npts,sizeof(unsigned char));
00556     else
00557         b = bpm;
00558     nbins = vircam_nint(highv - lowv + 1.0);
00559     histo = cpl_calloc(nbins,sizeof(*histo));
00560     nhist = 0;
00561     for (i = 0; i < npts; i++) {
00562         if (b[i] || data[i] < lowv || data[i] > highv)
00563             continue;
00564         ilev = vircam_nint(data[i] - lowv);
00565         ilev = max(0,min(nbins-1,ilev));
00566         histo[ilev] += 1;
00567         nhist += 1;
00568     }
00569     if (bpm == NULL)
00570         freespace(b);
00571 
00572     /* Right, find the median value and the first quartile. */
00573 
00574     iclip = nbins - 1;
00575     nhist2 = nhist;
00576     for (i = 0; i <= niter; i++) {
00577         halflev = (nhist2 + 1)/2;
00578         quartlev = (nhist2 + 3)/4;
00579         mlev = histexam(histo,nbins,halflev);
00580         *median = mlev + lowv;
00581         qlev = histexam(histo,nbins,quartlev);
00582         *sigma = (mlev - qlev)*1.48;
00583         if (i == niter)
00584             break;
00585         irej = 0;
00586         jst = vircam_nint(mlev + thresh*(*sigma));
00587         for (j = jst; j <= iclip; j++)
00588             irej += histo[j];
00589         if (irej == 0)
00590             break;
00591         iclip = jst - 1;
00592         nhist2 -= irej;
00593     }
00594     cpl_free(histo);
00595 }
00596 
00597 /*---------------------------------------------------------------------------*/
00623 /*---------------------------------------------------------------------------*/
00624 
00625 extern void vircam_medmad(float *data, unsigned char *bpm, long np, float *med,
00626                           float *mad) {
00627     int i;
00628     float *work;
00629 
00630     /* First find the median value */
00631 
00632     *med = vircam_med(data,bpm,np);
00633 
00634     /* Now work out the MAD. Start by getting a bit of workspace and filling
00635        it with absolute residuals */
00636     
00637     work = cpl_malloc(np*sizeof(*work));
00638     for (i = 0; i < np; i++)
00639         work[i] = (float)fabs((double)(data[i] - *med));
00640 
00641     /* Now get the median value of the absolute deviations */
00642 
00643     *mad = vircam_med(work,bpm,np);
00644 
00645     /* Tidy and exit */
00646 
00647     cpl_free(work);
00648 }
00649 
00650 /*---------------------------------------------------------------------------*/
00682 /*---------------------------------------------------------------------------*/
00683 
00684 extern void vircam_medmadcut(float *data, unsigned char *bpm, long np, 
00685                              float lcut, float hcut, float *med, float *mad) {
00686     int i;
00687     float *work;
00688     unsigned char *bad;
00689 
00690     /* Get a workspace for the pseudo-bad pixel mask... */
00691 
00692     bad = cpl_calloc(np,sizeof(*bad));
00693     if (bpm != NULL) {
00694         for (i = 0; i < np; i++)
00695             if (bpm[i] != 0 || data[i] < lcut || data[i] > hcut)
00696                 bad[i] = 1;
00697     } else {
00698         for (i = 0; i < np; i++) 
00699             if (data[i] < lcut || data[i] > hcut)
00700                 bad[i] = 1;
00701     }
00702 
00703     /* First find the median value */
00704 
00705     *med = vircam_med(data,bad,np);
00706     if (*med == CX_MAXFLOAT) {
00707         *mad = 0.0;
00708         cpl_free(bad);
00709         return;
00710     }
00711 
00712     /* Now work out the MAD. Start by getting a bit of workspace and filling
00713        it with absolute residuals */
00714     
00715     work = cpl_malloc(np*sizeof(*work));
00716     for (i = 0; i < np; i++)
00717         work[i] = (float)fabs((double)(data[i] - *med));
00718 
00719     /* Now get the median value of the absolute deviations */
00720 
00721     *mad = vircam_med(work,bad,np);
00722 
00723     /* Tidy and exit */
00724 
00725     cpl_free(work);
00726     cpl_free(bad);
00727 }
00728 
00729 /*---------------------------------------------------------------------------*/
00754 /*---------------------------------------------------------------------------*/
00755 
00756 extern void vircam_medsig(float *data, unsigned char *bpm, long np, float *med,
00757                           float *sig) {
00758     int i,n;
00759     float sum,resid;
00760 
00761     /* First find the median value */
00762 
00763     *med = vircam_med(data,bpm,np);
00764     if (*med == CX_MAXFLOAT) {
00765         *sig = 0.0;
00766         return;
00767     }
00768 
00769     /* If no bpm is present the just use them all */
00770 
00771     if (bpm == NULL) {
00772         sum = 0.0;
00773         for (i = 0; i < np; i++) {
00774             resid = data[i] - *med;
00775             sum += resid*resid;
00776         }
00777         *sig = sqrt(sum/(float)np);
00778 
00779     /* Otherwise test the bpm */
00780 
00781     } else {
00782         sum = 0.0;
00783         n = 0;
00784         for (i = 0; i < np; i++) {
00785             if (bpm[i] == 0) {
00786                 n++;
00787                 resid = data[i] - *med;
00788                 sum += resid*resid;
00789             }
00790         }
00791         if (n > 0) 
00792             *sig = sqrt(sum/(float)n);
00793         else
00794             *sig = 0.0;
00795     }
00796 }
00797 
00798 /*---------------------------------------------------------------------------*/
00820 /*---------------------------------------------------------------------------*/
00821 
00822 extern int vircam_sumbpm(unsigned char *bpm, int npts, int *sumb) {
00823     int j;
00824 
00825     *sumb = 0;
00826     for (j = 0; j < npts; j++)
00827         *sumb += bpm[j];
00828     return(VIR_OK);
00829 }
00830 
00833 static float histexam(int *histo, int nhist, int level) {
00834     int ilev,ii;
00835     float value;
00836 
00837     ii = 0;
00838     ilev = -1;
00839     while (ii < level && ilev < nhist-1)
00840         ii += histo[++ilev];
00841     value = (float)ilev - (float)(ii - level)/(float)histo[ilev] + 0.5;
00842     return(value);
00843 }
00844 
00845 /*
00846  * given an array a of n elements, return the element that would be at
00847  * position k, (0 <= k < n), if the array were sorted.  from Algorithms
00848  * and Data Structures in C++ by Leendert Ammeraal, pg. 82.  O(n).
00849  *
00850  * NB: partially reorders data in array
00851  */
00852 
00853 /* Stolen from C. Sabbey */
00854 
00855 
00856 static float kselect(float *a, int n, int k) {
00857     while (n > 1) {
00858         int i = 0, j = n - 1;
00859         float x = a[j/2], w;
00860 
00861         do {
00862             while (a[i] < x) i++;
00863             while (a[j] > x) j--;
00864             if (i < j) {
00865                 w = a[i]; a[i] = a[j]; a[j] = w;
00866             } else {
00867                 if (i == j) i++;
00868                 break;
00869             }
00870         } while (++i <= --j);
00871 
00872         if (k < i)
00873             n = i;
00874         else {
00875             a += i; n -= i; k -= i;
00876         }
00877     }
00878 
00879     return a[0];
00880 }
00881 
00882 static double dkselect(double *a, int n, int k) {
00883     while (n > 1) {
00884         int i = 0, j = n - 1;
00885         double x = a[j/2], w;
00886 
00887         do {
00888             while (a[i] < x) i++;
00889             while (a[j] > x) j--;
00890             if (i < j) {
00891                 w = a[i]; a[i] = a[j]; a[j] = w;
00892             } else {
00893                 if (i == j) i++;
00894                 break;
00895             }
00896         } while (++i <= --j);
00897 
00898         if (k < i)
00899             n = i;
00900         else {
00901             a += i; n -= i; k -= i;
00902         }
00903     }
00904 
00905     return a[0];
00906 }
00907 
00908 /*
00909 
00910 $Log: vircam_stats.c,v $
00911 Revision 1.20  2009/02/23 10:45:50  jim
00912 Modified vircam_med so that if no points are present, then CX_MAXFLOAT is
00913 returned
00914 
00915 Revision 1.19  2009/02/20 11:01:40  jim
00916 Plugged a memory leak
00917 
00918 Revision 1.18  2009/01/28 13:29:33  jim
00919 fixed typo
00920 
00921 Revision 1.17  2009/01/28 12:26:00  jim
00922 Trap for bad results from vircam_med in several routines
00923 
00924 Revision 1.16  2007/10/25 17:34:01  jim
00925 Modified to remove lint warnings
00926 
00927 Revision 1.15  2007/03/01 12:42:42  jim
00928 Modified slightly after code checking
00929 
00930 Revision 1.14  2007/02/06 11:59:17  jim
00931 Added vircam_medmadcut
00932 
00933 Revision 1.13  2006/12/19 13:29:06  jim
00934 Fixed medsig and medsigcut to take care of low number stats
00935 
00936 Revision 1.12  2006/09/08 09:22:19  jim
00937 Fixed vircam_qmedsig so that an input BPM could be NULL
00938 
00939 Revision 1.11  2006/06/06 13:06:48  jim
00940 Added vircam_medsig
00941 
00942 Revision 1.10  2006/05/26 19:33:29  jim
00943 Fixed a doc problem
00944 
00945 Revision 1.9  2006/05/24 13:36:48  jim
00946 Fixed warning message
00947 
00948 Revision 1.8  2006/05/08 14:53:16  jim
00949 Fixed problem negative variances by making sure that floats were cast to
00950 doubles properly
00951 
00952 Revision 1.6  2006/03/03 14:29:46  jim
00953 Modified definition of vir_fits and channel table
00954 
00955 Revision 1.5  2006/02/18 11:48:55  jim
00956 *** empty log message ***
00957 
00958 Revision 1.4  2006/01/23 10:30:49  jim
00959 Mainly documentation mods
00960 
00961 Revision 1.3  2005/12/14 22:17:33  jim
00962 Updated docs
00963 
00964 Revision 1.2  2005/10/14 13:19:13  jim
00965 Added vircam_medmad
00966 
00967 Revision 1.1.1.1  2005/08/05 08:29:09  jim
00968 Initial import
00969 
00970 
00971 */

Generated on 5 Mar 2013 for VIRCAM Pipeline by  doxygen 1.6.1