VIRCAM Pipeline  1.3.3
vircam_stats.c
1 /* $Id: vircam_stats.c,v 1.20 2009-02-23 10:45:50 jim Exp $
2  *
3  * This file is part of the VIRCAM Pipeline
4  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: jim $
23  * $Date: 2009-02-23 10:45:50 $
24  * $Revision: 1.20 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 /* Includes */
29 
30 #ifdef HAVE_CONFIG_H
31 #include <config.h>
32 #endif
33 
34 #include <math.h>
35 #include <string.h>
36 #include <cpl.h>
37 #include <cxtypes.h>
38 
39 #include "vircam_stats.h"
40 #include "vircam_utils.h"
41 
42 /* Static subroutine prototypes */
43 
44 static float kselect(float *a, int n, int k);
45 static double dkselect(double *a, int n, int k);
46 static float histexam(int *histo, int nhist, int level);
47 
61 /*---------------------------------------------------------------------------*/
87 /*---------------------------------------------------------------------------*/
88 
89 extern float vircam_med(float *data, unsigned char *bpm, long npts) {
90  int i,j,is_even,ilevel;
91  float *buf,value;
92 
93  /* Is there any point being here? */
94 
95  if (npts == 0)
96  return(CX_MAXFLOAT);
97 
98  /* If there is not BPM, then just do a straight forward median */
99 
100  buf = cpl_malloc(npts*sizeof(*buf));
101  if (bpm == NULL) {
102  is_even = !(npts & 1);
103  memmove((char *)buf,(char *)data,npts*sizeof(float));
104  if (is_even) {
105  ilevel = npts/2 - 1;
106  value = kselect(buf,npts,ilevel);
107  ilevel = npts/2;
108  value = 0.5*(value + kselect(buf,npts,ilevel));
109  } else {
110  ilevel = npts/2;
111  value = kselect(buf,npts,ilevel);
112  }
113 
114  /* Otherwise get rid of the dodgy values and then do the median */
115 
116  } else {
117  j = 0;
118  for (i = 0; i < npts; i++) {
119  if (bpm[i] == 0)
120  buf[j++] = data[i];
121  }
122  if (j == 0) {
123  cpl_free(buf);
124  value = CX_MAXFLOAT;
125  return(value);
126  }
127  is_even = !(j & 1);
128  if (is_even) {
129  ilevel = j/2 - 1;
130  value = kselect(buf,j,ilevel);
131  ilevel = j/2;
132  value = 0.5*(value + kselect(buf,j,ilevel));
133  } else {
134  ilevel = j/2;
135  value = kselect(buf,j,ilevel);
136  }
137  }
138  cpl_free(buf);
139  return(value);
140 }
141 
142 /*---------------------------------------------------------------------------*/
168 /*---------------------------------------------------------------------------*/
169 
170 extern double vircam_dmed(double *data, unsigned char *bpm, long npts) {
171  int i,j,is_even,ilevel;
172  double *buf,value;
173 
174  /* If there is not BPM, then just do a straight forward median */
175 
176  buf = cpl_malloc(npts*sizeof(*buf));
177  if (bpm == NULL) {
178  is_even = !(npts & 1);
179  memmove((char *)buf,(char *)data,npts*sizeof(double));
180  if (is_even) {
181  ilevel = npts/2 - 1;
182  value = dkselect(buf,npts,ilevel);
183  ilevel = npts/2;
184  value = 0.5*(value + dkselect(buf,npts,ilevel));
185  } else {
186  ilevel = npts/2;
187  value = dkselect(buf,npts,ilevel);
188  }
189 
190  /* Otherwise get rid of the dodgy values and then do the median */
191 
192  } else {
193  j = 0;
194  for (i = 0; i < npts; i++) {
195  if (bpm[i] == 0)
196  buf[j++] = data[i];
197  }
198  if (j == 0) {
199  cpl_free(buf);
200  value = CX_MAXDOUBLE;
201  return(value);
202  }
203  is_even = !(j & 1);
204  if (is_even) {
205  ilevel = j/2 - 1;
206  value = dkselect(buf,j,ilevel);
207  ilevel = j/2;
208  value = 0.5*(value + dkselect(buf,j,ilevel));
209  } else {
210  ilevel = j/2;
211  value = dkselect(buf,j,ilevel);
212  }
213  }
214  cpl_free(buf);
215  return(value);
216 }
217 
218 /*---------------------------------------------------------------------------*/
244 /*---------------------------------------------------------------------------*/
245 
246 extern float vircam_mean(float *data, unsigned char *bpm, long npts) {
247  int i,n;
248  float sum,value;
249 
250  /* Separate sections depending on whether there is a BPM or not */
251 
252  sum = 0.0;
253  if (bpm == NULL) {
254  n = npts;
255  for (i = 0; i < npts; i++)
256  sum += data[i];
257  } else {
258  n = 0;
259  for (i = 0; i < npts; i++) {
260  if (bpm[i] == 0) {
261  sum += data[i];
262  n++;
263  }
264  }
265  }
266  if (n > 0)
267  value = sum/(float)n;
268  else
269  value = CX_MAXFLOAT;
270  return(value);
271 }
272 
273 /*---------------------------------------------------------------------------*/
299 /*---------------------------------------------------------------------------*/
300 
301 extern double vircam_dmean(double *data, unsigned char *bpm, long npts) {
302  int i,n;
303  double sum,value;
304 
305  /* Separate sections depending on whether there is a BPM or not */
306 
307  sum = 0.0;
308  if (bpm == NULL) {
309  n = npts;
310  for (i = 0; i < npts; i++)
311  sum += data[i];
312  } else {
313  n = 0;
314  for (i = 0; i < npts; i++) {
315  if (bpm[i] == 0) {
316  sum += data[i];
317  n++;
318  }
319  }
320  }
321  if (n > 0)
322  value = sum/(float)n;
323  else
324  value = CX_MAXDOUBLE;
325  return(value);
326 }
327 
328 /*---------------------------------------------------------------------------*/
360 /*---------------------------------------------------------------------------*/
361 
362 extern int vircam_meansig(float *data, unsigned char *bpm, long npts,
363  float *mean, float *sig) {
364  int i,n;
365  double sum,sum2,d;
366  const char *fctid = "vircam_meansig";
367 
368  /* Separate sections depending on whether there is a BPM or not */
369 
370  sum = 0.0;
371  sum2 = 0.0;
372  if (bpm == NULL) {
373  n = npts;
374  for (i = 0; i < npts; i++) {
375  d = (double)(data[i]);
376  sum += d;
377  sum2 += d*d;
378  }
379  } else {
380  n = 0;
381  for (i = 0; i < npts; i++) {
382  if (bpm[i] == 0) {
383  d = (double)(data[i]);
384  sum += d;
385  sum2 += d*d;
386  n++;
387  }
388  }
389  }
390 
391  /* Check whether we can do the mean and sigma calculations */
392 
393  switch (n) {
394  case 0:
395  *mean = CX_MAXFLOAT;
396  *sig = CX_MAXFLOAT;
397  cpl_msg_warning(fctid,"All values flagged as bad\n");
398  return(VIR_WARN);
399  case 1:
400  *mean = (float)sum;
401  *sig = 0.0;
402  return(VIR_OK);
403  default:
404  sum /= (double)n;
405  *mean = (float)sum;
406  sum2 = sum2/(double)n - sum*sum;
407  *sig = (float)sqrt(max(1.0e-12,sum2));
408  return(VIR_OK);
409  }
410 }
411 
412 /*---------------------------------------------------------------------------*/
449 /*---------------------------------------------------------------------------*/
450 
451 extern int vircam_meansigcut(float *data, unsigned char *bpm, long npts,
452  float lcut, float hcut, float *mean, float *sig) {
453  int i,n;
454  double sum,sum2;
455  const char *fctid = "vircam_meansigcut";
456 
457  /* Separate sections depending on whether there is a BPM or not */
458 
459  sum = 0.0;
460  sum2 = 0.0;
461  if (bpm == NULL) {
462  n = 0;
463  for (i = 0; i < npts; i++) {
464  if (data[i] > lcut && data[i] < hcut) {
465  sum += data[i];
466  sum2 += data[i]*data[i];
467  n++;
468  }
469  }
470  } else {
471  n = 0;
472  for (i = 0; i < npts; i++) {
473  if (bpm[i] == 0 && data[i] > lcut && data[i] < hcut) {
474  sum += data[i];
475  sum2 += data[i]*data[i];
476  n++;
477  }
478  }
479  }
480 
481  /* Check whether we can do the mean and sigma calculations */
482 
483  switch (n) {
484  case 0:
485  *mean = CX_MAXFLOAT;
486  *sig = CX_MAXFLOAT;
487  cpl_msg_warning(fctid,"All values flagged as bad\n");
488  return(VIR_WARN);
489  case 1:
490  *mean = (float)sum;
491  *sig = 0.0;
492  return(VIR_OK);
493  default:
494  sum /= (double)n;
495  *mean = (float)sum;
496  sum2 = sum2/(double)n - sum*sum;
497  *sig = (float)sqrt(max(1.0e-12,sum2));
498  return(VIR_OK);
499  }
500 }
501 
502 /*---------------------------------------------------------------------------*/
540 /*---------------------------------------------------------------------------*/
541 
542 extern void vircam_qmedsig(float *data, unsigned char *bpm, long npts,
543  float thresh, int niter, float lowv, float highv,
544  float *median, float *sigma) {
545  int *histo,nbins,nhist,ilev,iclip,nhist2,halflev,quartlev;
546  int irej,jst,j;
547  long i;
548  float mlev,qlev;
549  unsigned char *b;
550 
551  /* Right, first thing is to histogram the data. Cut values below
552  and above the 'ceiling' values */
553 
554  if (bpm == NULL)
555  b = cpl_calloc(npts,sizeof(unsigned char));
556  else
557  b = bpm;
558  nbins = vircam_nint(highv - lowv + 1.0);
559  histo = cpl_calloc(nbins,sizeof(*histo));
560  nhist = 0;
561  for (i = 0; i < npts; i++) {
562  if (b[i] || data[i] < lowv || data[i] > highv)
563  continue;
564  ilev = vircam_nint(data[i] - lowv);
565  ilev = max(0,min(nbins-1,ilev));
566  histo[ilev] += 1;
567  nhist += 1;
568  }
569  if (bpm == NULL)
570  freespace(b);
571 
572  /* Right, find the median value and the first quartile. */
573 
574  iclip = nbins - 1;
575  nhist2 = nhist;
576  for (i = 0; i <= niter; i++) {
577  halflev = (nhist2 + 1)/2;
578  quartlev = (nhist2 + 3)/4;
579  mlev = histexam(histo,nbins,halflev);
580  *median = mlev + lowv;
581  qlev = histexam(histo,nbins,quartlev);
582  *sigma = (mlev - qlev)*1.48;
583  if (i == niter)
584  break;
585  irej = 0;
586  jst = vircam_nint(mlev + thresh*(*sigma));
587  for (j = jst; j <= iclip; j++)
588  irej += histo[j];
589  if (irej == 0)
590  break;
591  iclip = jst - 1;
592  nhist2 -= irej;
593  }
594  cpl_free(histo);
595 }
596 
597 /*---------------------------------------------------------------------------*/
623 /*---------------------------------------------------------------------------*/
624 
625 extern void vircam_medmad(float *data, unsigned char *bpm, long np, float *med,
626  float *mad) {
627  int i;
628  float *work;
629 
630  /* First find the median value */
631 
632  *med = vircam_med(data,bpm,np);
633 
634  /* Now work out the MAD. Start by getting a bit of workspace and filling
635  it with absolute residuals */
636 
637  work = cpl_malloc(np*sizeof(*work));
638  for (i = 0; i < np; i++)
639  work[i] = (float)fabs((double)(data[i] - *med));
640 
641  /* Now get the median value of the absolute deviations */
642 
643  *mad = vircam_med(work,bpm,np);
644 
645  /* Tidy and exit */
646 
647  cpl_free(work);
648 }
649 
650 /*---------------------------------------------------------------------------*/
682 /*---------------------------------------------------------------------------*/
683 
684 extern void vircam_medmadcut(float *data, unsigned char *bpm, long np,
685  float lcut, float hcut, float *med, float *mad) {
686  int i;
687  float *work;
688  unsigned char *bad;
689 
690  /* Get a workspace for the pseudo-bad pixel mask... */
691 
692  bad = cpl_calloc(np,sizeof(*bad));
693  if (bpm != NULL) {
694  for (i = 0; i < np; i++)
695  if (bpm[i] != 0 || data[i] < lcut || data[i] > hcut)
696  bad[i] = 1;
697  } else {
698  for (i = 0; i < np; i++)
699  if (data[i] < lcut || data[i] > hcut)
700  bad[i] = 1;
701  }
702 
703  /* First find the median value */
704 
705  *med = vircam_med(data,bad,np);
706  if (*med == CX_MAXFLOAT) {
707  *mad = 0.0;
708  cpl_free(bad);
709  return;
710  }
711 
712  /* Now work out the MAD. Start by getting a bit of workspace and filling
713  it with absolute residuals */
714 
715  work = cpl_malloc(np*sizeof(*work));
716  for (i = 0; i < np; i++)
717  work[i] = (float)fabs((double)(data[i] - *med));
718 
719  /* Now get the median value of the absolute deviations */
720 
721  *mad = vircam_med(work,bad,np);
722 
723  /* Tidy and exit */
724 
725  cpl_free(work);
726  cpl_free(bad);
727 }
728 
729 /*---------------------------------------------------------------------------*/
754 /*---------------------------------------------------------------------------*/
755 
756 extern void vircam_medsig(float *data, unsigned char *bpm, long np, float *med,
757  float *sig) {
758  int i,n;
759  float sum,resid;
760 
761  /* First find the median value */
762 
763  *med = vircam_med(data,bpm,np);
764  if (*med == CX_MAXFLOAT) {
765  *sig = 0.0;
766  return;
767  }
768 
769  /* If no bpm is present the just use them all */
770 
771  if (bpm == NULL) {
772  sum = 0.0;
773  for (i = 0; i < np; i++) {
774  resid = data[i] - *med;
775  sum += resid*resid;
776  }
777  *sig = sqrt(sum/(float)np);
778 
779  /* Otherwise test the bpm */
780 
781  } else {
782  sum = 0.0;
783  n = 0;
784  for (i = 0; i < np; i++) {
785  if (bpm[i] == 0) {
786  n++;
787  resid = data[i] - *med;
788  sum += resid*resid;
789  }
790  }
791  if (n > 0)
792  *sig = sqrt(sum/(float)n);
793  else
794  *sig = 0.0;
795  }
796 }
797 
798 /*---------------------------------------------------------------------------*/
820 /*---------------------------------------------------------------------------*/
821 
822 extern int vircam_sumbpm(unsigned char *bpm, int npts, int *sumb) {
823  int j;
824 
825  *sumb = 0;
826  for (j = 0; j < npts; j++)
827  *sumb += bpm[j];
828  return(VIR_OK);
829 }
830 
833 static float histexam(int *histo, int nhist, int level) {
834  int ilev,ii;
835  float value;
836 
837  ii = 0;
838  ilev = -1;
839  while (ii < level && ilev < nhist-1)
840  ii += histo[++ilev];
841  value = (float)ilev - (float)(ii - level)/(float)histo[ilev] + 0.5;
842  return(value);
843 }
844 
845 /*
846  * given an array a of n elements, return the element that would be at
847  * position k, (0 <= k < n), if the array were sorted. from Algorithms
848  * and Data Structures in C++ by Leendert Ammeraal, pg. 82. O(n).
849  *
850  * NB: partially reorders data in array
851  */
852 
853 /* Stolen from C. Sabbey */
854 
855 
856 static float kselect(float *a, int n, int k) {
857  while (n > 1) {
858  int i = 0, j = n - 1;
859  float x = a[j/2], w;
860 
861  do {
862  while (a[i] < x) i++;
863  while (a[j] > x) j--;
864  if (i < j) {
865  w = a[i]; a[i] = a[j]; a[j] = w;
866  } else {
867  if (i == j) i++;
868  break;
869  }
870  } while (++i <= --j);
871 
872  if (k < i)
873  n = i;
874  else {
875  a += i; n -= i; k -= i;
876  }
877  }
878 
879  return a[0];
880 }
881 
882 static double dkselect(double *a, int n, int k) {
883  while (n > 1) {
884  int i = 0, j = n - 1;
885  double x = a[j/2], w;
886 
887  do {
888  while (a[i] < x) i++;
889  while (a[j] > x) j--;
890  if (i < j) {
891  w = a[i]; a[i] = a[j]; a[j] = w;
892  } else {
893  if (i == j) i++;
894  break;
895  }
896  } while (++i <= --j);
897 
898  if (k < i)
899  n = i;
900  else {
901  a += i; n -= i; k -= i;
902  }
903  }
904 
905  return a[0];
906 }
907 
908 /*
909 
910 $Log: not supported by cvs2svn $
911 Revision 1.19 2009/02/20 11:01:40 jim
912 Plugged a memory leak
913 
914 Revision 1.18 2009/01/28 13:29:33 jim
915 fixed typo
916 
917 Revision 1.17 2009/01/28 12:26:00 jim
918 Trap for bad results from vircam_med in several routines
919 
920 Revision 1.16 2007/10/25 17:34:01 jim
921 Modified to remove lint warnings
922 
923 Revision 1.15 2007/03/01 12:42:42 jim
924 Modified slightly after code checking
925 
926 Revision 1.14 2007/02/06 11:59:17 jim
927 Added vircam_medmadcut
928 
929 Revision 1.13 2006/12/19 13:29:06 jim
930 Fixed medsig and medsigcut to take care of low number stats
931 
932 Revision 1.12 2006/09/08 09:22:19 jim
933 Fixed vircam_qmedsig so that an input BPM could be NULL
934 
935 Revision 1.11 2006/06/06 13:06:48 jim
936 Added vircam_medsig
937 
938 Revision 1.10 2006/05/26 19:33:29 jim
939 Fixed a doc problem
940 
941 Revision 1.9 2006/05/24 13:36:48 jim
942 Fixed warning message
943 
944 Revision 1.8 2006/05/08 14:53:16 jim
945 Fixed problem negative variances by making sure that floats were cast to
946 doubles properly
947 
948 Revision 1.6 2006/03/03 14:29:46 jim
949 Modified definition of vir_fits and channel table
950 
951 Revision 1.5 2006/02/18 11:48:55 jim
952 *** empty log message ***
953 
954 Revision 1.4 2006/01/23 10:30:49 jim
955 Mainly documentation mods
956 
957 Revision 1.3 2005/12/14 22:17:33 jim
958 Updated docs
959 
960 Revision 1.2 2005/10/14 13:19:13 jim
961 Added vircam_medmad
962 
963 Revision 1.1.1.1 2005/08/05 08:29:09 jim
964 Initial import
965 
966 
967 */