OMEGA Pipeline Reference Manual  1.0.5
omega_mdome.c
1 /* $Id: omega_mdome.c,v 1.7 2012-08-30 07:46:52 agabasch Exp $
2  *
3  * This file is part of the OMEGA Pipeline
4  * Copyright (C) 2002,2003 European Southern Observatory
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19  */
20 
21 /*
22  * $Author: agabasch $
23  * $Date: 2012-08-30 07:46:52 $
24  * $Revision: 1.7 $
25  * $Name: not supported by cvs2svn $
26  */
27 
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31 
32 
33 #include "omega_recipe.h"
34 #include "omega_background.h"
35 
36 /*----------------------------------------------------------------------------*/
67 /*-----------------------------------------------------------------------------
68  Functions prototypes
69  -----------------------------------------------------------------------------*/
70 
71 static int omega_mdome_create(cpl_plugin *);
72 static int omega_mdome_exec(cpl_plugin *);
73 static int omega_mdome_destroy(cpl_plugin *);
74 static int omega_mdome(cpl_frameset *, cpl_parameterlist *);
75 
76 /*-----------------------------------------------------------------------------
77  Private Functions
78  -----------------------------------------------------------------------------*/
79 
80 int omega_mdome_combine(cpl_parameterlist *pars, int xn);
81 static void omega_mdome_init(void);
82 static void omega_mdome_tidy(void);
83 
84 /*-----------------------------------------------------------------------------
85  Static structures
86  -----------------------------------------------------------------------------*/
87 
88 static struct {
89  /* Inputs. Parameters */
90  int extnum;
91  int oc;
92  int paf;
93  double sigma;
94  double lthre;
95  double hthre;
96 
97  /* Outputs. QC parameters */
98  int CountColdPixels;
99  double Mean;
100  double Median;
101  double Stdev;
102  double RawMin;
103  double RawMax;
104  double RawMean;
105  double RawMedian;
106  double RawStdev;
107 
108 }omega_mdome_config;
109 
110 /* Input and Output */
111 static struct {
112 
113  cpl_size *labels;
114  const cpl_frame *mbframe;
115  const cpl_frame *cframe;
116  const cpl_frame *hframe;
117  cpl_frameset *domelist;
118  cpl_stats *stats;
119  cpl_propertylist *ph;
120  cpl_propertylist *eh;
121  omega_fits *firstdome;
122 
123  /* Products */
124  cpl_image *mdome;
125  cpl_image *cpixels;
126 
127 }ps;
128 
129 /*-----------------------------------------------------------------------------
130  Static global variables
131  -----------------------------------------------------------------------------*/
132 
133 /* Static variables */
134 #define RECIPE "omega_mdome"
135 
136 static int isfirst;
137 
138 
139 /*----------------------------------------------------------------------------*/
147 /*----------------------------------------------------------------------------*/
148 int cpl_plugin_get_info(cpl_pluginlist * list)
149 {
150  cpl_recipe * recipe = cpl_calloc(1, sizeof(*recipe)) ;
151  cpl_plugin * plugin = &recipe->interface ;
152 
153  cpl_plugin_init(plugin,
154  CPL_PLUGIN_API,
155  OMEGA_BINARY_VERSION,
156  CPL_PLUGIN_TYPE_RECIPE,
157  "omega_mdome",
158  "OMEGA - Create Master Dome Flat for each chip (Calfile 542).",
159  "This recipe is used to derive a valid domeflat frame (Calfile 542) \n"
160  "for one particular chip and filter. The recipe always takes as input \n"
161  "a list of raw domeflat frames, a master bias and a measurement of the \n"
162  "gain of the chip under consideration. In addition, a hot pixel map \n"
163  "(Calfile 522) and a cold pixel map (Calfile 535) can be provided. Optionally, \n"
164  "an overscan correction mode can be set. The default is to apply no overscan \n"
165  "correction.\n\n"
166  "The raw dome flats are trimmed and overscan corrected. The data are then \n"
167  "normalized, averaged, and the result is normalized again. Image statisics \n"
168  "are computed for the resulting frame.",
169  "Sandra Castro",
170  "scastro@eso.org",
172  omega_mdome_create,
173  omega_mdome_exec,
174  omega_mdome_destroy) ;
175 
176  cpl_pluginlist_append(list, plugin) ;
177 
178  return 0;
179 }
180 
181 /*----------------------------------------------------------------------------*/
190 /*----------------------------------------------------------------------------*/
191 static int omega_mdome_create(cpl_plugin * plugin)
192 {
193  cpl_recipe * recipe;
194  cpl_parameter * p ;
195  char *path = NULL;
196 
197  /* Do not create the recipe if an error code is already set */
198  if (cpl_error_get_code() != CPL_ERROR_NONE) {
199  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
200  cpl_func, __LINE__, cpl_error_get_where());
201  return (int)cpl_error_get_code();
202  }
203 
204  if (plugin == NULL) {
205  cpl_msg_error(cpl_func, "Null plugin");
206  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
207  }
208 
209  /* Verify plugin type */
210  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
211  cpl_msg_error(cpl_func, "Plugin is not a recipe");
212  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
213  }
214 
215  /* Get the recipe */
216  recipe = (cpl_recipe *)plugin;
217 
218  /* Create the parameters list in the cpl_recipe object */
219  recipe->parameters = cpl_parameterlist_new() ;
220 
221  if (recipe->parameters == NULL) {
222  cpl_msg_error(cpl_func, "Parameter list allocation failed");
223  cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT);
224  }
225 
226  /* Fill the parameters list */
227 
228  /* General Parameters */
229  p = cpl_parameter_new_value("omega.omega_mdome.ExtensionNumber",
230  CPL_TYPE_INT,
231  "FITS extension number to load (1 to 32). (-1 = all)",
232  "omega_mdome",
233  -1) ;
234 
235  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"ext") ;
236  cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
237  cpl_parameterlist_append(recipe->parameters, p) ;
238 
239 
240  p = cpl_parameter_new_range("omega.omega_mdome.OverscanMethod",
241  CPL_TYPE_INT,
242  "Overscan Correction Method",
243  "omega_mdome",
244  0, 0, 6);
245 
246  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"oc-meth") ;
247  cpl_parameterlist_append(recipe->parameters, p) ;
248 
249  p = cpl_parameter_new_value("omega.omega_mdome.PAF",
250  CPL_TYPE_BOOL,
251  "Boolean value to create PAF files. 1(Yes), 0(No)",
252  "omega_mdome",
253  1) ;
254 
255  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "paf") ;
256  cpl_parameterlist_append(recipe->parameters, p) ;
257 
258  p = cpl_parameter_new_value("omega.omega_mdome.SigmaClip",
259  CPL_TYPE_DOUBLE,
260  "Sigma Clipping Threshold",
261  "omega_mdome",
262  3.0) ;
263 
264  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sig-clip") ;
265  cpl_parameterlist_append(recipe->parameters, p) ;
266 
267 
268 /*------------ Parameters for creating a Cold Pixels Map---------------- */
269  p = cpl_parameter_new_range("omega.omega_mdome.LowThre",
270  CPL_TYPE_DOUBLE,
271  "Low flagging threshold for cold pixels map",
272  "omega_coldpixels",
273  0.93, 0.90, 1.00) ;
274 
275  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "low") ;
276  cpl_parameterlist_append(recipe->parameters, p) ;
277 
278  p = cpl_parameter_new_range("omega.omega_mdome.HighThre",
279  CPL_TYPE_DOUBLE,
280  "High flagging threshold for cold pixels map",
281  "omega_coldpixels",
282  1.07, 1.00, 1.10) ;
283 
284  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"high") ;
285  cpl_parameterlist_append(recipe->parameters, p) ;
286 
287  p = cpl_parameter_new_range("omega.omega_mdome.BackSize",
288  CPL_TYPE_INT,
289  "Sextractor background mesh size for cold pixels map",
290  "omega_coldpixels",
291  32, 4, 256) ;
292 
293  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"backsize") ;
294  cpl_parameterlist_append(recipe->parameters, p) ;
295 
296 /* ------------------ PATH to external executable programs --------------*/
297  path = cpl_sprintf("%s", OMEGA_BIN_PATH);
298  p = cpl_parameter_new_value("omega.omega_mdome.BinPath",
299  CPL_TYPE_STRING,
300  "Path to any external executable program.",
301  "omega.BinPath",
302  path);
303 
304  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "bin-path");
305  cpl_parameterlist_append(recipe->parameters, p);
306  cpl_free(path);
307 
308 /* ------------------ Sextractor parameters ------------------------- */
309  path = cpl_sprintf("%s/omega.sex", OMEGA_CONFIG_PATH);
310  p = cpl_parameter_new_value("omega.omega_mdome.SexConfig",
311  CPL_TYPE_STRING,
312  "Path to Sextractor config file.",
313  "omega.Sextractor",
314  path);
315 
316  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sex-config");
317  cpl_parameterlist_append(recipe->parameters, p);
318  cpl_free(path);
319 
320  path = cpl_sprintf("%s/omega.conv", OMEGA_CONFIG_PATH);
321  p = cpl_parameter_new_value("omega.omega_mdome.SexConv",
322  CPL_TYPE_STRING,
323  "Path to Sextractor convolution mask file.",
324  "omega.Sextractor",
325  path);
326 
327  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"sex-conv");
328  cpl_parameterlist_append(recipe->parameters, p);
329  cpl_free(path);
330 
331 
332  path = cpl_sprintf("%s/omega.param", OMEGA_CONFIG_PATH);
333  p = cpl_parameter_new_value("omega.omega_mdome.SexParam",
334  CPL_TYPE_STRING,
335  "Path to Sextractor parameters file.",
336  "omega.Sextractor",
337  path);
338 
339  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "sex-param");
340  cpl_parameterlist_append(recipe->parameters, p);
341  cpl_free(path);
342 
343  path = cpl_sprintf("%s/omega.nnw", OMEGA_CONFIG_PATH);
344  p = cpl_parameter_new_value("omega.omega_mdome.SexNnw",
345  CPL_TYPE_STRING,
346  "Path to Sextractor neural network config file.",
347  "omega.Sextractor",
348  path);
349 
350  cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI,"sex-nnw");
351  cpl_parameterlist_append(recipe->parameters, p);
352  cpl_free(path);
353 
354  p = cpl_parameter_new_value("omega.omega_mdome.BackThreshold",
355  CPL_TYPE_DOUBLE,
356  "Detection threshold for background in image (Sextractor DETECT_THRESH)",
357  "omega_Sextractor",
358  1000.) ;
359 
360  cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI, "backthre") ;
361  cpl_parameterlist_append(recipe->parameters, p) ;
362 
363 
364 
365  return 0;
366 }
367 
368 /*----------------------------------------------------------------------------*/
374 /*----------------------------------------------------------------------------*/
375 static int omega_mdome_exec(cpl_plugin * plugin)
376 {
377  cpl_recipe * recipe;
378  int recipe_status;
379 /* cpl_errorstate initial_errorstate = cpl_errorstate_get();*/
380 
381  /* Return immediately if an error code is already set */
382  if (cpl_error_get_code() != CPL_ERROR_NONE) {
383  cpl_msg_error(cpl_func, "%s():%d: An error is already set: %s",
384  cpl_func, __LINE__, cpl_error_get_where());
385  return (int)cpl_error_get_code();
386  }
387 
388  if (plugin == NULL) {
389  cpl_msg_error(cpl_func, "Null plugin");
390  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
391  }
392 
393  /* Verify plugin type */
394  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
395  cpl_msg_error(cpl_func, "Plugin is not a recipe");
396  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
397  }
398 
399  /* Get the recipe */
400  recipe = (cpl_recipe *)plugin;
401 
402  /* Verify parameter and frame lists */
403  if (recipe->parameters == NULL) {
404  cpl_msg_error(cpl_func, "Recipe invoked with NULL parameter list");
405  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
406  }
407  if (recipe->frames == NULL) {
408  cpl_msg_error(cpl_func, "Recipe invoked with NULL frame set");
409  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
410  }
411 
412  /* Invoke the recipe */
413  recipe_status = omega_mdome(recipe->frames, recipe->parameters);
414 
415  /* Ensure DFS-compliance of the products */
416  if (cpl_dfs_update_product_header(recipe->frames)) {
417  if (!recipe_status) recipe_status = (int)cpl_error_get_code();
418  }
419 
420  /* Dump the error history since recipe execution start.
421  At this point the recipe cannot recover from the error */
422  /* if (!cpl_errorstate_is_equal(initial_errorstate)) {
423  cpl_errorstate_dump(initial_errorstate, CPL_FALSE, NULL);
424  }*/
425 
426  return recipe_status;
427 
428 }
429 
430 /*----------------------------------------------------------------------------*/
436 /*----------------------------------------------------------------------------*/
437 static int omega_mdome_destroy(cpl_plugin * plugin)
438 {
439  cpl_recipe *recipe;
440 
441  if (plugin == NULL) {
442  cpl_msg_error(cpl_func, "Null plugin");
443  cpl_ensure_code(0, (int)CPL_ERROR_NULL_INPUT);
444  }
445 
446  /* Verify plugin type */
447  if (cpl_plugin_get_type(plugin) != CPL_PLUGIN_TYPE_RECIPE) {
448  cpl_msg_error(cpl_func, "Plugin is not a recipe");
449  cpl_ensure_code(0, (int)CPL_ERROR_TYPE_MISMATCH);
450  }
451 
452  /* Get the recipe */
453  recipe = (cpl_recipe *)plugin;
454 
455  cpl_parameterlist_delete(recipe->parameters);
456 
457  return 0 ;
458 }
459 
460 /*----------------------------------------------------------------------------*/
467 /*----------------------------------------------------------------------------*/
468 static int omega_mdome(cpl_frameset *set, cpl_parameterlist *pars)
469 {
470 
471  int j,jst,jfn;
472  cpl_size nlab;
473  int nflats = 0;
474  int oscan1 = 0;
475  int status = 1;
476 
477  float bias = BIAS;
478 
479  char *outcpm = NULL;
480  char *outmdome = NULL;
481  const char *_id = "omega_mdome";
482  cpl_frame *firstframe = NULL;
483  cpl_frame *prframe_cpm = NULL;
484  cpl_frame *prframe_mdome = NULL;
485  const cpl_frame *refframe = NULL;
486  cpl_parameter *par = NULL;
487  cpl_propertylist *qclist = NULL;
488  cpl_propertylist *alist = NULL;
489  cpl_stats *diffstats = NULL;
490 
491 
492  /*Start the recipe */
493 
494  if (!pars) {
495  cpl_msg_error (_id, "Parameters list not found");
496  return -1;
497  }
498 
499  if (cpl_frameset_is_empty(set) == 1) {
500  cpl_msg_error (_id, "Frameset not found");
501  return -1;
502  }
503 
504 /* Retrieve input parameters */
505  par = cpl_parameterlist_find(pars, "omega.omega_mdome.ExtensionNumber") ;
506  omega_mdome_config.extnum = cpl_parameter_get_int(par) ;
507 
508  par = cpl_parameterlist_find(pars, "omega.omega_mdome.OverscanMethod") ;
509  omega_mdome_config.oc = cpl_parameter_get_int(par) ;
510 
511  par = cpl_parameterlist_find(pars, "omega.omega_mdome.SigmaClip") ;
512  omega_mdome_config.sigma = cpl_parameter_get_double(par) ;
513 
514  par = cpl_parameterlist_find(pars, "omega.omega_mdome.LowThre") ;
515  omega_mdome_config.lthre = cpl_parameter_get_double(par) ;
516 
517  par = cpl_parameterlist_find(pars, "omega.omega_mdome.HighThre") ;
518  omega_mdome_config.hthre = cpl_parameter_get_double(par) ;
519 
520  par = cpl_parameterlist_find(pars, "omega.omega_mdome.PAF") ;
521  omega_mdome_config.paf = cpl_parameter_get_bool(par) ;
522 
523 /* Identify the RAW and CALIB frames in the input frameset */
524  if (oc_dfs_set_groups(set)) {
525  cpl_msg_error(_id, "Cannot identify RAW and CALIB frames") ;
526  return -1 ;
527  }
528 
529  /*Initialized things*/
530  omega_mdome_init();
531 
532 /* Verify the frameset contents. */
533  if ((ps.labels = cpl_frameset_labelise(set,omega_compare_tags,
534  &nlab)) == NULL) {
535  cpl_msg_error(_id,"Cannot labelise the input frameset");
536  omega_mdome_tidy();
537  return -1;
538  }
539  if ((ps.domelist = omega_frameset_subgroup(set,ps.labels,nlab,
540  MDOME_RAW)) == NULL) {
541  cpl_msg_error(_id,"Cannot find dome frames in input frameset");
542  omega_mdome_tidy();
543  return -1;
544  }
545 
546 
547  nflats = cpl_frameset_count_tags(ps.domelist, MDOME_RAW);
548  if (nflats < 2) {
549  cpl_msg_error (_id, "Need at least 2 (%s) frames to run "
550  "this recipe", MDOME_RAW);
551  omega_mdome_tidy();
552  return -1;
553  }
554 
555  cpl_msg_info (_id,"There are %d %s in frame set",nflats, MDOME_RAW);
556 
557  /* Check for calibration frames */
558  /* Master Bias */
559  ps.mbframe = cpl_frameset_find_const(set, OMEGA_CALIB_BIAS);
560  if (ps.mbframe == NULL) {
561  cpl_msg_info(_id,"A Master Bias is not present in frame set. Using default value %f", bias);
562  }
563  else{
564  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_BIAS, cpl_frame_get_filename(ps.mbframe));
565  }
566 
567  /* Cold Pixels Map */
568  ps.cframe = cpl_frameset_find_const(set, OMEGA_CALIB_CPM);
569  if(ps.cframe != NULL){
570  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_CPM, cpl_frame_get_filename(ps.cframe));
571  }
572 
573  /* Hot Pixels Map */
574  ps.hframe = cpl_frameset_find_const(set, OMEGA_CALIB_HPM);
575  if(ps.hframe != NULL){
576  cpl_msg_info(_id,"Using %s %s",OMEGA_CALIB_HPM, cpl_frame_get_filename(ps.hframe));
577  }
578 
579  /* Optional Reference master dome frame */
580  refframe = cpl_frameset_find_const(set, REFDOME);
581  if (refframe != NULL)
582  cpl_msg_info(cpl_func,"Using %s for comparison",cpl_frame_get_filename(refframe));
583 
584  /* Get first frame from frame set */
585  firstframe = cpl_frameset_get_first(ps.domelist);
586 
587  /* Loop for each of the image extensions */
588  omega_exten_range(omega_mdome_config.extnum,&jst,&jfn);
589  if(omega_mdome_config.extnum == 0){
590  cpl_msg_error(cpl_func,"Unsupported extension request, %d",omega_mdome_config.extnum);
591  omega_mdome_tidy();
592  return -1;
593  }
594 
595  /* Check which instrument */
596  if (omega_pfits_check_instrume(firstframe) == 1)
597  sprintf(INSTRUME,"wfi");
598 
599  /* Check which extensions */
600  if(omega_pfits_check_instrume(firstframe) == 1 &&
601  omega_mdome_config.extnum == 0 && jfn == 32)
602  jfn = 8;
603 
604  for (j = jst; j <= jfn; j++) {
605 
606  isfirst = (j == jst);
607  cpl_msg_info(_id,"Beginning work on extension %d",j);
608  omega_mdome_config.CountColdPixels = 0;
609  omega_mdome_config.Mean = 0.0;
610  omega_mdome_config.Median = 0.0;
611  omega_mdome_config.Stdev = 0.0;
612  omega_mdome_config.RawMin = 0.0;
613  omega_mdome_config.RawMax = 0.0;
614  omega_mdome_config.RawMean = 0.0;
615  omega_mdome_config.RawMedian = 0.0;
616  omega_mdome_config.RawStdev = 0.0;
617 
618  /* Check overscan correction method consistency */
619  if(ps.mbframe != NULL){
620  oscan1 = omega_pfits_get_overscan(ps.mbframe, j);
621  if(oscan1 != omega_mdome_config.oc) {
622  cpl_msg_warning (_id, "Overscan correction mode for Master Bias (oc = %d) differs from "
623  "the one used here (oc = %d)", oscan1, omega_mdome_config.oc);
624  }
625  }
626 
627 
628  /* Load first dome flat */
629  ps.firstdome = omega_fits_load(firstframe,CPL_TYPE_FLOAT,j);
630  ps.eh = omega_fits_get_ehu(ps.firstdome);
631 
632  /* Call the combining routine */
633  status = omega_mdome_combine(pars, j);
634  if(status == 1){
635  cpl_msg_warning(_id, "Image detector is not live");
636  /* Save dummy product */
637  freefits(ps.firstdome);
638  ps.eh = NULL;
639  continue;
640  }
641  else if(status == -1){
642  cpl_msg_error(_id,"Cannot combine images");
643  freespace(outcpm);
644  freespace(outmdome);
645  omega_mdome_tidy();
646  return -1;
647  }
648 
649  qclist = cpl_propertylist_new();
650 
651  /* Compare with reference frame */
652  if(refframe != NULL){
653  if(omega_compare_reference(ps.mdome,refframe,j,&diffstats) == -1){
654  cpl_msg_warning(cpl_func,"Cannot compare with reference frame");
655  }
656  else{
657  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME MEAN",
658  cpl_stats_get_mean(diffstats));
659  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME MEAN",
660  "Mean of difference with reference");
661  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME MEDIAN",
662  cpl_stats_get_median(diffstats));
663  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME MEDIAN",
664  "Median of difference with reference");
665  cpl_propertylist_append_double(qclist,"ESO QC DIFF REFDOME STDEV",
666  cpl_stats_get_stdev(diffstats));
667  cpl_propertylist_set_comment(qclist,"ESO QC DIFF REFDOME STDEV",
668  "Stdev of difference with reference");
669 
670  freestats(diffstats);
671  }
672  }
673 
674  /* Save the products */
675  /* Save the MASTER DOME product */
676 
677  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME MEAN",
678  omega_mdome_config.Mean) ;
679  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME MEAN",
680  "Mean of master dome flat");
681 
682  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME MEDIAN",
683  omega_mdome_config.Median) ;
684  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME MEDIAN",
685  "Median of master dome flat");
686 
687  cpl_propertylist_append_double(qclist, "ESO QC MASTER DOME STDEV",
688  omega_mdome_config.Stdev) ;
689  cpl_propertylist_set_comment (qclist, "ESO QC MASTER DOME STDEV",
690  "Std Deviation of master dome flat");
691 
692  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MIN",
693  omega_mdome_config.RawMin);
694  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MAX",
695  omega_mdome_config.RawMax);
696  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MEAN",
697  omega_mdome_config.RawMean);
698  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME MEDIAN",
699  omega_mdome_config.RawMedian);
700  cpl_propertylist_append_double(qclist, "ESO QC RAW DOME STDEV",
701  omega_mdome_config.RawStdev);
702 
703  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MIN",
704  "median value of the raw dome flat having the lowest flux");
705  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MAX",
706  "median value of the raw dome flat having the highest flux");
707  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MEAN",
708  "mean value of all input raw dome flats (ADU)");
709  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME MEDIAN",
710  "median value of all input raw dome flats (ADU)");
711  cpl_propertylist_set_comment(qclist, "ESO QC RAW DOME STDEV",
712  "standard deviation of all input raw dome flats (ADU)");
713 
714  if(isfirst){
715  outmdome = cpl_sprintf("%s_%s.fits", INSTRUME,MDOME_PROCATG);
716  prframe_mdome = omega_product_frame(outmdome, MDOME_PROCATG, CPL_FRAME_TYPE_IMAGE);
717  }
718 
719  alist = cpl_propertylist_new();
720  cpl_propertylist_append_string(alist, "EXTNAME",
721  cpl_propertylist_get_string(ps.eh, "EXTNAME"));
722  cpl_propertylist_set_comment(alist,"EXTNAME", "Extension name");
723 
724  /* Add DRS keywords */
725  cpl_propertylist_update_int(alist, "ESO DRS OVERSCAN METHOD", omega_mdome_config.oc);
726  cpl_propertylist_set_comment(alist, "ESO DRS OVERSCAN METHOD", "overscan correction method");
727 
728  cpl_propertylist_copy_property_regexp(alist, ps.eh, WCS_KEYS, 0);
729 
730  if(omega_save_image(ps.mdome,set,pars,alist,qclist,CPL_BPP_IEEE_FLOAT,outmdome,
731  RECIPE,prframe_mdome,NULL,isfirst) == -1){
732  cpl_msg_error(_id,"Cannot save product %s", MDOME_PROCATG);
733  freeplist(qclist);
734  freeplist(alist);
735  freespace(outcpm);
736  freespace(outmdome);
737  omega_mdome_tidy();
738  return -1;;
739  }
740  freeplist(qclist);
741  freeimage(ps.mdome);
742 
743  /* Save the COLD PIXELS MAP product */
744  qclist = cpl_propertylist_new();
745  cpl_propertylist_append_int(qclist,"ESO QC NUMBER COLD PIXELS",
746  omega_mdome_config.CountColdPixels);
747 
748  cpl_propertylist_set_comment(qclist,"ESO QC NUMBER COLD PIXELS",
749  "Number of cold pixels");
750  if(isfirst){
751  outcpm = cpl_sprintf("%s_%s.fits", INSTRUME,CPM_PROCATG);
752  prframe_cpm = omega_product_frame(outcpm, CPM_PROCATG, CPL_FRAME_TYPE_IMAGE);
753  }
754 
755  if(omega_save_image(ps.cpixels,set,pars,alist,qclist,CPL_BPP_16_SIGNED,outcpm,
756  RECIPE,prframe_cpm,NULL,isfirst) == -1){
757  cpl_msg_error(_id,"Cannot save product %s", CPM_PROCATG);
758  freeplist(qclist);
759  freeplist(alist);
760  freespace(outcpm);
761  freespace(outmdome);
762  omega_mdome_tidy();
763  return -1;;
764  }
765 
766  freeplist(qclist);
767  freeplist(alist);
768  freeimage(ps.cpixels);
769  freefits(ps.firstdome);
770  ps.firstdome = NULL;
771  ps.eh = NULL;
772 
773  } /* go and work on next extension */
774 
775  /*Clean up */
776 
777  freespace(outcpm);
778  freespace(outmdome);
779  omega_mdome_tidy();
780 
781  return 0;
782 
783 }
784 
785 /*----------------------------------------------------------------------------*/
796 /*----------------------------------------------------------------------------*/
797 int omega_mdome_combine(cpl_parameterlist *pars, int xn)
798 {
799 
800  int i,nflats,live, naxis1, naxis2;
801  float bias = BIAS;
802  double gain = 0.0;
803  double median = 1.0;
804  double threshold = 0.0;
805  double *data_scales = NULL;
806  const char *_id = "omega_mdome_combine";
807  const char *mdome_name = "omega_mdome_temp.fits";
808  const char *backname = "omega_mdome_background.fits";
809 
810  cpl_error_code code;
811  const cpl_frame *domefr = NULL;
812  cpl_vector *scales = NULL;
813  cpl_vector *median_vector = NULL;
814  cpl_image *mbias = NULL;
815  cpl_image *trim_raw = NULL, *back_image = NULL, *dev = NULL;
816  cpl_image *good_float = NULL, *good_int = NULL, *median_all = NULL;
817  cpl_image *sum_data = NULL, *sum_good = NULL, *new_image = NULL;
818  cpl_image *norm_image = NULL;
819  cpl_imagelist *ilist = NULL;
820  cpl_mask *good = NULL, *bpm_map = NULL;
821  cpl_mask *pixelmap = NULL;
822 
823  nflats = cpl_frameset_get_size(ps.domelist);
824  domefr = cpl_frameset_get_first_const(ps.domelist);
825 
826  /* Check that this detector is live*/
827  omega_pfits_get_detlive(ps.eh,&live);
828  if (! live) {
829  return 1;
830  }
831 
832  /* Load master bias image */
833  if(ps.mbframe != NULL){
834  mbias = cpl_image_load(cpl_frame_get_filename(ps.mbframe), CPL_TYPE_FLOAT,0,xn);
835  if (mbias == NULL) {
836  cpl_msg_warning(_id,"Cannot load image %s", OMEGA_CALIB_BIAS);
837  }
838  }
839 
840  /*Create Mask from Hot and Cold Map */
841  bpm_map = makebpm(ps.hframe, ps.cframe, xn);
842 
843 
844  scales = cpl_vector_new (nflats);
845  median_vector = cpl_vector_new (nflats);
846  cpl_vector_fill(median_vector,0.0);
847 
848  data_scales = cpl_vector_get_data(scales);
849 
850  ilist = cpl_imagelist_new();
851  cpl_msg_info (_id,"Doing trim and overscan correction on images");
852 
853  /* Loop through all images in frame list */
854  for (i=0; i< nflats; i++){
855 
856  trim_raw = TrimOscanCorrect(domefr, omega_mdome_config.oc, xn);
857  if(trim_raw == NULL){
858  freevector(scales);
859  freevector(median_vector);
860  freeilist(ilist);
861  return -1;
862  }
863 
864  if (mbias != NULL)
865  cpl_image_subtract(trim_raw, mbias);
866  else
867  cpl_image_subtract_scalar(trim_raw, bias);
868 
869  if (bpm_map == NULL) {
870  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
871  median = cpl_stats_get_median(ps.stats);
872  }
873  else {
874  cpl_image_reject_from_mask(trim_raw, bpm_map);
875  ps.stats = cpl_stats_new_from_image(trim_raw, CPL_STATS_ALL);
876  median = cpl_stats_get_median(ps.stats);
877  }
878 
879  cpl_vector_set(median_vector, i, median);
880 
881  data_scales[i] = (double)1.0/median;
882 
883  cpl_image_divide_scalar(trim_raw, median);
884 
885  cpl_imagelist_set(ilist, trim_raw, i);
886 
887 
888  domefr = cpl_frameset_get_next_const(ps.domelist);
889  if (domefr == NULL)
890  break;
891 
892  freestats(ps.stats);
893 
894  }
895 
896  omega_mdome_config.RawMin = cpl_vector_get_min(median_vector);
897  omega_mdome_config.RawMax = cpl_vector_get_max(median_vector);
898  omega_mdome_config.RawMean = cpl_vector_get_mean(median_vector);
899  omega_mdome_config.RawMedian = cpl_vector_get_median(median_vector);
900  omega_mdome_config.RawStdev = cpl_vector_get_stdev(median_vector);
901 
902  freevector(median_vector);
903 
904  freestats(ps.stats);
905  freeimage(mbias);
906  freemask(bpm_map);
907 
908  if (ilist == NULL) {
909  cpl_msg_error(_id,"Error in image list <%s>",cpl_error_get_message());
910  freevector(scales);
911  freeimage(trim_raw);
912  return -1;
913  }
914 
915 
916  /* Read gain from the header */
917  /* FIXME: or read gain from gain recipe product?? */
918 
919  /* The images have the CONAD as the correct value for
920  * the gain, therefore the following lines should be
921  * changed if the headers are fixed.
922  */
923 
924 // omega_pfits_get_gain(ps.eh, &gain);
925  omega_pfits_get_conad(ps.eh, &gain);
926  /*FIXME: should we avoid having gain=0?*/
927  if (gain < 0.1){
928  omega_pfits_get_gain(ps.eh, &gain);
929 // omega_pfits_get_conad(ps.eh, &gain);
930  if(gain < 0.1) {
931  cpl_msg_info(_id,"Using default value of gain");
932  gain = 2.0;
933  }
934  }
935  else {
936  cpl_msg_info(_id,"Gain value from image is %g", gain);
937  }
938 
939  cpl_msg_info (_id,"Computing the median of all images...");
940  median_all = cpl_imagelist_collapse_median_create(ilist);
941 
942  if (median_all == NULL) {
943  cpl_msg_error (_id,"Cannot take median of list <%s>",cpl_error_get_message());
944  freevector(scales);
945  freeilist(ilist);
946  return -1;
947  }
948 
949  naxis1 = cpl_image_get_size_x(median_all);
950  naxis2 = cpl_image_get_size_y(median_all);
951 
952  sum_data = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
953  sum_good = cpl_image_new(naxis1,naxis2,CPL_TYPE_FLOAT);
954 
955  /* Clip the outliers from the images*/
956  for (i=0; i< nflats; i++){
957 
958  trim_raw = cpl_imagelist_get(ilist, i);
959  new_image = cpl_image_duplicate(trim_raw);
960 
961  /* FIXME: setting negative values to 0 to avoid failure of SQRT in next step
962  * when there are negative values in the image. These negative values happen when any
963  * given pixel in the domeflat is smaller than the same pixel in the master_bias image.
964  */
965  cpl_image_threshold(trim_raw, 0, FLT_MAX, 0, 2e20);
966 
967  dev = cpl_image_subtract_create(median_all, trim_raw);
968 
969  code = cpl_image_power(trim_raw, 0.5);
970  if(code != CPL_ERROR_NONE) {
971  cpl_msg_error(_id,"Error in SQRT operation <%s>",cpl_error_get_message());
972  freevector(scales);
973  freeilist(ilist);
974  freeimage(median_all);
975  freeimage(sum_data);
976  freeimage(sum_good);
977  freeimage(new_image);
978  freeimage(dev);
979  return -1;
980  }
981 
982  cpl_image_divide(dev, trim_raw);
983 
984  threshold = omega_mdome_config.sigma * (sqrt(gain * data_scales[i]));
985 
986  good = cpl_mask_threshold_image_create(dev, -threshold, threshold);
987 
988  freeimage(dev);
989  good_int = cpl_image_new_from_mask(good) ;
990  freemask(good) ;
991 
992  good_float = cpl_image_cast(good_int, CPL_TYPE_FLOAT);
993  freeimage(good_int);
994 
995  cpl_image_multiply(new_image, good_float);
996  cpl_image_add(sum_data, new_image);
997 
998  cpl_image_add(sum_good, good_float);
999 
1000  freeimage(new_image);
1001  freeimage(good_float);
1002 
1003  }
1004 
1005  freeimage(median_all);
1006  freevector(scales);
1007 
1008  /* Create master dome flat */
1009  ps.mdome = cpl_image_divide_create(sum_data, sum_good);
1010  if (ps.mdome == NULL) {
1011  cpl_msg_error(_id,"Error in division %s <%s>",MDOME_PROCATG, cpl_error_get_message());
1012  freeilist(ilist);
1013  freeimage(sum_data);
1014  freeimage(sum_good);
1015  return -1;
1016  }
1017 
1018  freeimage(sum_data);
1019  freeimage(sum_good);
1020  freeilist(ilist);
1021 
1022  /* Save temporary master dome to disk to use in Sextractor */
1023  cpl_image_save(ps.mdome, mdome_name, BITPIX, NULL, CPL_IO_DEFAULT);
1024 
1025 
1026  /* Create New Cold Pixels Map */
1027  /* Call function to create background image*/
1028  if(omega_create_background(pars, mdome_name, backname) != 0){
1029  cpl_msg_warning(_id,"Error in creating background image");
1030 /* return -1;*/
1031  }
1032 
1033  back_image = cpl_image_load(backname, CPL_TYPE_FLOAT, 0, 0);
1034  if (back_image != NULL){
1035  norm_image = cpl_image_divide_create(ps.mdome, back_image);
1036  freeimage(back_image);
1037  }
1038  else{
1039  cpl_msg_warning(_id,"Could not create normalised image. Cannot load background image.");
1040  norm_image = NULL;
1041  }
1042 
1043  /*Explanation for next steps: the image is normalized to around 1.0, therefore
1044  the bad pixels will be all pixels OUTSIDE the range lthre--hthre. Because of
1045  this, a mask_not needs to be applied to set the bad pixels to CPL_BINARY_1
1046  and count them using cpl_mask_count (which counts CPL_BINARY_1)*/
1047 
1048  /* This cpl function creates a mask of all the pixels within the given interval.
1049  * For example if an image has pixels in the interval 0.0 - 1.0 and the function
1050  * is called like this cpl_mask_threshold_image_create(img,0.5,1.0), all the pixels
1051  * in the range 0.5 to 1.0 will be set to CPL_BINARY_1. The other pixels,
1052  * < 0.5 and > 1.0 will be set to CPL_BINARY_0.
1053  */
1054  pixelmap = cpl_mask_threshold_image_create(norm_image, omega_mdome_config.lthre,
1055  omega_mdome_config.hthre);
1056  cpl_mask_not(pixelmap);
1057 
1058  cpl_msg_info(_id,"Creating pixel map with thresholds: %g,%g",omega_mdome_config.lthre,
1059  omega_mdome_config.hthre);
1060 
1061  omega_mdome_config.CountColdPixels = cpl_mask_count(pixelmap);
1062  if (omega_mdome_config.CountColdPixels == -1) {
1063  cpl_msg_warning(_id,"Pixelmap is NULL. <%s>", cpl_error_get_message());
1064  }
1065 
1066  cpl_msg_info(_id,"Detected %d cold pixels", omega_mdome_config.CountColdPixels);
1067 
1068  freeimage(norm_image);
1069 
1070  /* Convert mask to image to save later */
1071  ps.cpixels = cpl_image_new_from_mask(pixelmap);
1072  freemask(pixelmap);
1073 
1074  /* Compute statistics on master dome flat */
1075  ps.stats = cpl_stats_new_from_image(ps.mdome, CPL_STATS_ALL);
1076  if(ps.stats == NULL && (cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND)){
1077  cpl_msg_warning(_id," There are no good pixels in image for doing statistics");
1078  cpl_error_reset();
1079  }
1080  else{
1081  omega_mdome_config.Mean = cpl_stats_get_mean(ps.stats);
1082  omega_mdome_config.Median = cpl_stats_get_median(ps.stats);
1083  omega_mdome_config.Stdev = cpl_stats_get_stdev(ps.stats);
1084  }
1085 
1086  freestats(ps.stats);
1087 
1088 
1089  return 0;
1090 }
1091 
1092 
1093 /* Initialize the pointers */
1094 static void omega_mdome_init(void) {
1095  ps.labels = NULL;
1096  ps.mbframe = NULL;
1097  ps.cframe = NULL;
1098  ps.hframe = NULL;
1099  ps.domelist = NULL;
1100  ps.ph = NULL;
1101  ps.eh = NULL;
1102  ps.stats = NULL;
1103  ps.mdome = NULL;
1104  ps.cpixels = NULL;
1105  ps.firstdome = NULL;
1106 }
1107 
1108 /* Free any allocated memory */
1109 static void omega_mdome_tidy(void) {
1110  freespace(ps.labels);
1111  freeframeset(ps.domelist);
1112  freestats(ps.stats);
1113  freeimage(ps.mdome);
1114  freeimage(ps.cpixels);
1115  freefits(ps.firstdome);
1116 }