HAWKI Pipeline Reference Manual 1.8.12
hawki_cal_flat.c
00001 /* $Id: hawki_cal_flat.c,v 1.25 2012/01/11 15:02:18 cgarcia Exp $
00002  *
00003  * This file is part of the HAWKI Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: cgarcia $
00023  * $Date: 2012/01/11 15:02:18 $
00024  * $Revision: 1.25 $
00025  * $Name: hawki-1_8_12 $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033                                 Includes
00034  -----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 #include <cpl.h>
00039 
00040 #include "irplib_utils.h"
00041 
00042 #include "hawki_utils.h"
00043 #include "hawki_image_stats.h"
00044 #include "hawki_load.h"
00045 #include "hawki_save.h"
00046 #include "hawki_pfits.h"
00047 #include "hawki_dfs.h"
00048 
00049 /*-----------------------------------------------------------------------------
00050                             Functions prototypes
00051  -----------------------------------------------------------------------------*/
00052 
00053 static int hawki_cal_flat_create(cpl_plugin *) ;
00054 static int hawki_cal_flat_exec(cpl_plugin *) ;
00055 static int hawki_cal_flat_destroy(cpl_plugin *) ;
00056 static int hawki_cal_flat(cpl_parameterlist *, cpl_frameset *) ;
00057 
00058 static int hawki_cal_flat_retrieve_input_param
00059 (cpl_parameterlist  *  parlist);
00060 static cpl_imagelist ** hawki_cal_flat_reduce(
00061         cpl_frameset    *   flatframes,
00062         const char      *   dark_file,
00063         cpl_table       **  raw_flat_stats,
00064         cpl_table       **  raw_flat_odd_column_stats,
00065         cpl_table       **  raw_flat_even_column_stats,
00066         cpl_table       **  raw_flat_odd_row_stats,
00067         cpl_table       **  raw_flat_even_row_stats,
00068         cpl_vector      **  selected); 
00069 static int hawki_cal_flat_clean_outliers(cpl_image *, cpl_imagelist *,
00070         cpl_imagelist *, cpl_vector *, cpl_image **) ;
00071 static int hawki_cal_flat_save
00072 (cpl_imagelist     ** flat,
00073  cpl_table         ** raw_flat_stats,
00074  cpl_table         ** raw_flat_odd_column_stats,
00075  cpl_table         ** raw_flat_even_column_stats,
00076  cpl_table         ** raw_flat_odd_row_stats,
00077  cpl_table         ** raw_flat_even_row_stats,
00078  cpl_vector        ** raw_selected,
00079  int                  set_nb,
00080  const cpl_frame   *  bpmdark,
00081  cpl_frameset      *  flatframes,
00082  cpl_frameset      *  calibframes,
00083  cpl_parameterlist *  parlist,
00084  cpl_frameset      *  set_tot);
00085 static int hawki_cal_flat_compare(const cpl_frame *, const cpl_frame *) ;
00086 static cpl_imagelist * hawki_cal_flat_merge_bpms
00087 (const cpl_frame *   bpm_orig,
00088  cpl_imagelist   *   bpm_to_add);
00089 static int hawki_cal_flat_select
00090 (cpl_vector  *   meds,
00091  cpl_vector  *   rms,
00092  int             auto_flag,
00093  int             auto_max_bins,
00094  double          min_level,
00095  double          max_level,
00096  double          max_rms,
00097  int             min_nframes,
00098  cpl_vector  *   selection);
00099 static cpl_vector * hawki_cal_flat_extract_vector(cpl_vector *,
00100         cpl_vector *) ;
00101 
00102 /*-----------------------------------------------------------------------------
00103                             Static variables
00104  -----------------------------------------------------------------------------*/
00105 
00106 static struct {
00107     /* Inputs */
00108     int         llx ;
00109     int         lly ;
00110     int         urx ;
00111     int         ury ;
00112     int         normalise ;
00113     int         second_pass ;
00114     double      sigma_badres ;
00115     double      sigma_bpm ;
00116     double      lowval_bpm ;
00117     double      highval_bpm ;
00118     int         select_auto ;
00119     int         select_auto_max_bins;
00120     double      select_min_level[HAWKI_NB_DETECTORS];
00121     double      select_max_level[HAWKI_NB_DETECTORS];
00122     double      select_max_rms[HAWKI_NB_DETECTORS];
00123     int         select_min_nframes ;
00124     int         extra_stats;
00125 } hawki_cal_flat_config ;
00126 
00127 static struct {
00128     /* Outputs */
00129     int         nb_badpix[HAWKI_NB_DETECTORS];
00130     double      norm[HAWKI_NB_DETECTORS];
00131     double      med_stdev[HAWKI_NB_DETECTORS];
00132     double      med_avg[HAWKI_NB_DETECTORS];
00133     double      med_med[HAWKI_NB_DETECTORS];
00134     double      med_min[HAWKI_NB_DETECTORS];
00135     double      med_max[HAWKI_NB_DETECTORS];
00136 } hawki_cal_flat_outputs;
00137 
00138 static char hawki_cal_flat_description[] = 
00139 "hawki_cal_flat -- HAWKĪI imaging flat-field creation from twillight images.\n"
00140 "The input of the recipe files listed in the Set Of Frames (sof-file)\n"
00141 "must be tagged as:\n"
00142 "raw-file.fits "HAWKI_CAL_FLAT_RAW" or\n"
00143 "Optional inputs are:\n"
00144 "bpmdark-file.fits "HAWKI_CALPRO_BPM_HOT"\n"
00145 "dark-file.fits "HAWKI_CALPRO_DARK"\n"
00146 "dark_err-file.fits "HAWKI_CALPRO_DARK_ERR"\n"
00147 "The recipe creates as an output:\n"
00148 "hawki_cal_flat_setxx.fits ("HAWKI_CALPRO_FLAT"): Master flat for filter xx\n"
00149 "hawki_cal_flat_err_setxx.fits ("HAWKI_CALPRO_FLAT_ERRMAP"): Master flat residuals\n"
00150 "hawki_cal_flat_bpmflat_setxx.fits ("HAWKI_CALPRO_BPM_COLD"): BPM from the flat\n"
00151 "hawki_cal_flat_stats_setxx.fits ("HAWKI_CALPRO_FLAT_STATS"): Stats of the individual flats\n"
00152 "Optionally it also creates:\n"
00153 "hawki_cal_flat_bpm_setxx.fits ("HAWKI_CALPRO_BPM"): Bad pixel mask combining bpm from dark and flat\n"
00154 "hawki_cal_flat_stats_ec_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_EVEN_COL"): Stats of the individual flats for even columns\n"
00155 "hawki_cal_flat_stats_oc_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_ODD_COL"): Stats of the individual flats for odd columns\n"
00156 "hawki_cal_flat_stats_er_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_EVEN_ROW"): Stats of the individual flats for even rows\n"
00157 "hawki_cal_flat_stats_or_setxx.fits ("HAWKI_CALPRO_FLAT_STATS_ODD_ROW"): Stats of the individual flats for odd rows\n"
00158 "Return code:\n"
00159 "esorex exits with an error code of 0 if the recipe completes successfully\n"
00160 "or 1 otherwise";
00161 
00162 
00163 
00164 
00165 
00166 /*-----------------------------------------------------------------------------
00167                                 Functions code
00168  -----------------------------------------------------------------------------*/
00169 
00170 /*----------------------------------------------------------------------------*/
00178 /*----------------------------------------------------------------------------*/
00179 int cpl_plugin_get_info(cpl_pluginlist * list)
00180 {
00181     cpl_recipe  *   recipe = cpl_calloc(1, sizeof(*recipe)) ;
00182     cpl_plugin  *   plugin = &recipe->interface ;
00183 
00184     cpl_plugin_init(plugin,
00185                     CPL_PLUGIN_API,
00186                     HAWKI_BINARY_VERSION,
00187                     CPL_PLUGIN_TYPE_RECIPE,
00188                     "hawki_cal_flat",
00189                     "Twillight flat recipe",
00190                     hawki_cal_flat_description,
00191                     "Cesar Enrique Garcia Dabo",
00192                     PACKAGE_BUGREPORT,  
00193                     hawki_get_license(),
00194                     hawki_cal_flat_create,
00195                     hawki_cal_flat_exec,
00196                     hawki_cal_flat_destroy) ;
00197 
00198     cpl_pluginlist_append(list, plugin) ;
00199     
00200     return 0;
00201 }
00202 
00203 /*----------------------------------------------------------------------------*/
00212 /*----------------------------------------------------------------------------*/
00213 static int hawki_cal_flat_create(cpl_plugin * plugin)
00214 {
00215     cpl_recipe      * recipe ;
00216     cpl_parameter   * p ;
00217 
00218     /* Get the recipe out of the plugin */
00219     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00220         recipe = (cpl_recipe *)plugin ;
00221     else return -1 ;
00222 
00223     /* Create the parameters list in the cpl_recipe object */
00224     recipe->parameters = cpl_parameterlist_new() ;
00225 
00226     /* Fill the parameters list */
00227     /* --zone */
00228     p = cpl_parameter_new_value("hawki.hawki_cal_flat.zone",
00229                                 CPL_TYPE_STRING,
00230                                 "Stats zone",
00231                                 "hawki.hawki_cal_flat",
00232                                 "1,1,2048,2048") ;
00233     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "zone") ;
00234     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00235     cpl_parameterlist_append(recipe->parameters, p) ;
00236 
00237     /* --normalise */
00238     p = cpl_parameter_new_value("hawki.hawki_cal_flat.normalise",
00239             CPL_TYPE_BOOL, "Flag to apply the normalisation",
00240             "hawki.hawki_cal_flat", FALSE) ;
00241     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "normalise") ;
00242     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00243     cpl_parameterlist_append(recipe->parameters, p) ;
00244 
00245     /* --second_pass */
00246     p = cpl_parameter_new_value("hawki.hawki_cal_flat.second_pass",
00247             CPL_TYPE_BOOL, "Flag to apply a second pass computation",
00248             "hawki.hawki_cal_flat", TRUE) ;
00249     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "second_pass") ;
00250     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00251     cpl_parameterlist_append(recipe->parameters, p) ;
00252 
00253     /* --sigma_badres */
00254     p = cpl_parameter_new_value("hawki.hawki_cal_flat.sigma_badres",
00255             CPL_TYPE_DOUBLE, "sigma for detection of bad flat results",
00256             "hawki.hawki_cal_flat", 1.0) ;
00257     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sigma_badres") ;
00258     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00259     cpl_parameterlist_append(recipe->parameters, p) ;
00260 
00261     /* --sigma_bpm */
00262     p = cpl_parameter_new_value("hawki.hawki_cal_flat.sigma_bpm",
00263             CPL_TYPE_DOUBLE, "sigma for detection of bad pixels",
00264             "hawki.hawki_cal_flat", 10.0) ;
00265     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sigma_bpm") ;
00266     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00267     cpl_parameterlist_append(recipe->parameters, p) ;
00268  
00269     /* --lowval_bpm */
00270     p = cpl_parameter_new_value("hawki.hawki_cal_flat.lowval_bpm",
00271             CPL_TYPE_DOUBLE, "values of the flat below this will be included "
00272             "in the bpm. In units of final flat (normalised if normalise is on)",
00273             "hawki.hawki_cal_flat", .1) ;
00274     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "lowval_bpm") ;
00275     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00276     cpl_parameterlist_append(recipe->parameters, p) ;
00277  
00278     /* --highval_bpm */
00279     p = cpl_parameter_new_value("hawki.hawki_cal_flat.highval_bpm",
00280             CPL_TYPE_DOUBLE, "values of the flat above this will be included "
00281             "in the bpm. In units of final flat (normalized if normalise is on)",
00282             "hawki.hawki_cal_flat", 10.0) ;
00283     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "highval_bpm") ;
00284     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00285     cpl_parameterlist_append(recipe->parameters, p) ;
00286  
00287     /* --select_auto */
00288     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_auto",
00289             CPL_TYPE_BOOL, "Flag to automatically select the good input frames",
00290             "hawki.hawki_cal_flat", TRUE);
00291     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_auto") ;
00292     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00293     cpl_parameterlist_append(recipe->parameters, p) ;
00294 
00295     /* --select_auto_max_bins */
00296     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_auto_max_bins",
00297             CPL_TYPE_INT, "Maximum number of frames requested",
00298             "hawki.hawki_cal_flat", 10) ;
00299     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_auto_max_bins");
00300     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00301     cpl_parameterlist_append(recipe->parameters, p) ;
00302  
00303     /* --select_min_level */
00304     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_min_level",
00305             CPL_TYPE_STRING, "Minimum ADU level for frames selection",
00306             "hawki.hawki_cal_flat", "-1.0") ;
00307     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_min_level") ;
00308     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00309     cpl_parameterlist_append(recipe->parameters, p) ;
00310  
00311     /* --select_max_level */
00312     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_max_level",
00313             CPL_TYPE_STRING, "Maximum ADU level for frames selection",
00314             "hawki.hawki_cal_flat", "25000");
00315     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_max_level");
00316     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00317     cpl_parameterlist_append(recipe->parameters, p);
00318  
00319     /* --select_max_rms */
00320     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_max_rms",
00321             CPL_TYPE_STRING, "Maximum RMS for frames selection",
00322             "hawki.hawki_cal_flat", "4000");
00323     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_max_rms");
00324     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00325     cpl_parameterlist_append(recipe->parameters, p);
00326  
00327     /* --select_min_nframes */
00328     p = cpl_parameter_new_value("hawki.hawki_cal_flat.select_min_nframes",
00329             CPL_TYPE_INT, "Minimum number of frames requested",
00330             "hawki.hawki_cal_flat", 3);
00331     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "select_min_nframes") ;
00332     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00333     cpl_parameterlist_append(recipe->parameters, p) ;
00334  
00335     /* --extra_stats */
00336     p = cpl_parameter_new_value("hawki.hawki_cal_flat.extra_stats",
00337             CPL_TYPE_BOOL, "Request for even/odd column/rows statistics",
00338             "hawki.hawki_cal_flat", FALSE) ;
00339     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "extra_stats") ;
00340     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV) ;
00341     cpl_parameterlist_append(recipe->parameters, p) ;
00342 
00343     /* Return */
00344     return 0;
00345 }
00346 
00347 /*----------------------------------------------------------------------------*/
00353 /*----------------------------------------------------------------------------*/
00354 static int hawki_cal_flat_exec(cpl_plugin * plugin)
00355 {
00356     cpl_recipe  *   recipe ;
00357 
00358     /* Get the recipe out of the plugin */
00359     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00360         recipe = (cpl_recipe *)plugin ;
00361     else return -1 ;
00362 
00363     /* Issue a banner */
00364     hawki_print_banner();
00365 
00366     return hawki_cal_flat(recipe->parameters, recipe->frames) ;
00367 }
00368 
00369 /*----------------------------------------------------------------------------*/
00375 /*----------------------------------------------------------------------------*/
00376 static int hawki_cal_flat_destroy(cpl_plugin * plugin)
00377 {
00378     cpl_recipe  *   recipe ;
00379 
00380     /* Get the recipe out of the plugin */
00381     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00382         recipe = (cpl_recipe *)plugin ;
00383     else return -1 ;
00384 
00385     cpl_parameterlist_delete(recipe->parameters) ;
00386     return 0 ;
00387 }
00388 
00389 /*----------------------------------------------------------------------------*/
00396 /*----------------------------------------------------------------------------*/
00397 static int hawki_cal_flat(
00398         cpl_parameterlist   *   parlist, 
00399         cpl_frameset        *   framelist)
00400 {
00401     cpl_size        *   labels ;
00402     cpl_size            nlabels ;
00403     cpl_frameset    *   flatframes ;
00404     int                 nflats;
00405     const char      *   dark ;
00406     const char      *   dark_err;
00407     const cpl_frame *   bpmdark ;
00408     cpl_imagelist   **  twflat ;
00409     cpl_table       **  raw_flat_stats;
00410     cpl_table       **  raw_flat_odd_column_stats = NULL;
00411     cpl_table       **  raw_flat_even_column_stats = NULL;
00412     cpl_table       **  raw_flat_odd_row_stats = NULL;
00413     cpl_table       **  raw_flat_even_row_stats = NULL;
00414     cpl_vector      **  raw_selected;
00415     cpl_size            i;
00416     int                 j ;
00417     cpl_errorstate      error_prevstate = cpl_errorstate_get();
00418     
00419     /* Retrieve input parameters */
00420     if(hawki_cal_flat_retrieve_input_param(parlist))
00421     {
00422         cpl_msg_error(__func__, "Wrong parameters");
00423         return -1;
00424     }
00425     
00426     /* Identify the RAW and CALIB frames in the input frameset */
00427     if (hawki_dfs_set_groups(framelist)) {
00428         cpl_msg_error(__func__, "Cannot identify RAW and CALIB frames") ;
00429         return -1;
00430     }
00431 
00432     /* Retrieve raw frames */
00433     if ((flatframes = hawki_extract_frameset(framelist,
00434                     HAWKI_CAL_FLAT_RAW)) == NULL) {
00435         cpl_msg_error(__func__, "Cannot find flat frames in the input list (%s)",
00436                       HAWKI_CAL_FLAT_RAW);
00437         return -1 ;
00438     }
00439 
00440     /* Retrieve calibration frames */
00441     bpmdark = cpl_frameset_find_const(framelist, HAWKI_CALPRO_BPM_HOT);
00442     dark = hawki_extract_first_filename(framelist, HAWKI_CALPRO_DARK);
00443     dark_err = hawki_extract_first_filename(framelist, HAWKI_CALPRO_DARK_ERR);
00444 
00445     /* Labelise all input flat frames */
00446     labels = cpl_frameset_labelise(flatframes, hawki_cal_flat_compare, 
00447                 &nlabels);
00448     if (labels == NULL) {
00449         cpl_msg_error(__func__, "Cannot labelise input frames") ;
00450         cpl_frameset_delete(flatframes);
00451         return -1;
00452     }
00453    
00454     /* Extract sets and reduce each of them */
00455     for (i=0 ; i<nlabels ; i++) 
00456     {
00457         cpl_frameset    *   this_filter_flats;
00458         
00459         /* Reduce data set nb i */
00460         cpl_msg_info(__func__, "Reduce data set no %d out of %d", i+1, nlabels);
00461         cpl_msg_indent_more() ;
00462         this_filter_flats = cpl_frameset_extract(flatframes, labels, i) ;
00463         nflats = cpl_frameset_get_size(this_filter_flats);
00464         
00465         /* Allocate and initialize statistics */
00466         raw_flat_stats = cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
00467         raw_selected = cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_vector*));
00468         for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00469         {
00470             raw_selected[j] = cpl_vector_new(nflats);
00471             raw_flat_stats[j] = cpl_table_new(nflats);
00472         }
00473         /* Initialize the statistics table */
00474         hawki_image_stats_initialize(raw_flat_stats);
00475         if(hawki_cal_flat_config.extra_stats)
00476         {
00477             raw_flat_odd_column_stats = 
00478                 cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
00479             raw_flat_even_column_stats = 
00480                 cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
00481             raw_flat_odd_row_stats = 
00482                 cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
00483             raw_flat_even_row_stats = 
00484                 cpl_malloc(HAWKI_NB_DETECTORS*sizeof(cpl_table*));
00485             for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00486             {
00487                 raw_flat_odd_column_stats[j] = cpl_table_new(nflats);
00488                 raw_flat_even_column_stats[j] = cpl_table_new(nflats);
00489                 raw_flat_odd_row_stats[j] = cpl_table_new(nflats);
00490                 raw_flat_even_row_stats[j] = cpl_table_new(nflats);
00491             }
00492             /* Initialize the statistics table */
00493             hawki_image_stats_initialize(raw_flat_odd_column_stats);
00494             hawki_image_stats_initialize(raw_flat_even_column_stats);
00495             hawki_image_stats_initialize(raw_flat_odd_row_stats);
00496             hawki_image_stats_initialize(raw_flat_even_row_stats);
00497         }
00498 
00499         /* Reduce */
00500         if ((twflat = hawki_cal_flat_reduce
00501                 (this_filter_flats, 
00502                  dark,
00503                  raw_flat_stats,
00504                  raw_flat_odd_column_stats,
00505                  raw_flat_even_column_stats,
00506                  raw_flat_odd_row_stats,
00507                  raw_flat_even_row_stats,
00508                  raw_selected)) == NULL) 
00509         {
00510             for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00511             {
00512                 cpl_table_delete(raw_flat_stats[j]);
00513                 cpl_vector_delete(raw_selected[j]);
00514             }
00515             cpl_free(raw_flat_stats);
00516             cpl_free(raw_selected);
00517             if(hawki_cal_flat_config.extra_stats)
00518             {
00519                 for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00520                 {
00521                     cpl_table_delete(raw_flat_odd_column_stats[j]);
00522                     cpl_table_delete(raw_flat_even_column_stats[j]);
00523                     cpl_table_delete(raw_flat_odd_row_stats[j]);
00524                     cpl_table_delete(raw_flat_even_row_stats[j]);
00525                 }
00526                 cpl_free(raw_flat_odd_column_stats);
00527                 cpl_free(raw_flat_even_column_stats);
00528                 cpl_free(raw_flat_odd_row_stats);
00529                 cpl_free(raw_flat_even_row_stats);
00530             }
00531             cpl_frameset_delete(this_filter_flats);
00532             cpl_frameset_delete(flatframes);
00533             cpl_free(labels);
00534             cpl_msg_error(__func__, "Cannot reduce set nb %d", i+1) ;
00535             return 1;
00536 
00537         } else {
00538             /* Save the products */
00539             cpl_frameset    *   calib_frames;
00540 
00541             cpl_msg_info(__func__, "Save the products") ;
00542             calib_frames = cpl_frameset_new();
00543             if(bpmdark)
00544                 cpl_frameset_insert(calib_frames, cpl_frame_duplicate(bpmdark));
00545             if(dark)
00546                 cpl_frameset_insert(calib_frames,
00547                         cpl_frame_duplicate(cpl_frameset_find_const(framelist, 
00548                                 HAWKI_CALPRO_DARK)));
00549             if(dark_err)
00550                 cpl_frameset_insert(calib_frames,
00551                         cpl_frame_duplicate(cpl_frameset_find_const(framelist, 
00552                                 HAWKI_CALPRO_DARK_ERR)));
00553             hawki_cal_flat_save
00554                 (twflat, raw_flat_stats, 
00555                  raw_flat_odd_column_stats,
00556                  raw_flat_even_column_stats,
00557                  raw_flat_odd_row_stats,
00558                  raw_flat_even_row_stats,
00559                  raw_selected,
00560                  i+1, bpmdark, this_filter_flats, calib_frames, 
00561                  parlist, framelist);
00562             cpl_imagelist_delete(twflat[0]);
00563             cpl_imagelist_delete(twflat[1]);
00564             cpl_imagelist_delete(twflat[2]);
00565             if (hawki_cal_flat_config.second_pass)
00566                 cpl_imagelist_delete(twflat[3]);
00567             cpl_free(twflat);
00568             for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00569             {
00570                 cpl_table_delete(raw_flat_stats[j]);
00571                 cpl_vector_delete(raw_selected[j]);
00572             }
00573             cpl_free(raw_flat_stats);
00574             cpl_free(raw_selected);
00575             if(hawki_cal_flat_config.extra_stats)
00576             {
00577                 for (j=0 ; j<HAWKI_NB_DETECTORS ; j++)
00578                 {
00579                     cpl_table_delete(raw_flat_odd_column_stats[j]);
00580                     cpl_table_delete(raw_flat_even_column_stats[j]);
00581                     cpl_table_delete(raw_flat_odd_row_stats[j]);
00582                     cpl_table_delete(raw_flat_even_row_stats[j]);
00583                 }
00584                 cpl_free(raw_flat_odd_column_stats);
00585                 cpl_free(raw_flat_even_column_stats);
00586                 cpl_free(raw_flat_odd_row_stats);
00587                 cpl_free(raw_flat_even_row_stats);
00588             }
00589             cpl_frameset_delete(calib_frames);            
00590         }
00591         cpl_msg_indent_less();
00592         cpl_frameset_delete(this_filter_flats);
00593     }
00594     
00595     if(!cpl_errorstate_is_equal(error_prevstate))
00596         cpl_msg_warning(__func__,"Probably some data could not be saved. "
00597                                  "Check permisions or disk space");
00598     
00599     
00600     /* Free and return */
00601     cpl_frameset_delete(flatframes);
00602     cpl_free(labels); 
00603 
00604     /* Return */
00605     if (cpl_error_get_code()) return -1 ;
00606     else return 0;
00607 }
00608 
00609 /*----------------------------------------------------------------------------*/
00620 /*----------------------------------------------------------------------------*/
00621 static cpl_imagelist ** hawki_cal_flat_reduce(
00622         cpl_frameset    *   flatframes,
00623         const char      *   dark_file,
00624         cpl_table       **  raw_flat_stats,
00625         cpl_table       **  raw_flat_odd_column_stats,
00626         cpl_table       **  raw_flat_even_column_stats,
00627         cpl_table       **  raw_flat_odd_row_stats,
00628         cpl_table       **  raw_flat_even_row_stats,
00629         cpl_vector      **  selected) 
00630 {
00631     int                     nima ;
00632     cpl_image           *   ima_cur ;
00633     cpl_image           *   big_ima ;
00634     cpl_image           *   big_badres ;
00635     cpl_vector          *   medians[HAWKI_NB_DETECTORS];
00636     cpl_vector          *   stdevs[HAWKI_NB_DETECTORS];
00637     cpl_vector          *   sub_medians ;
00638     cpl_imagelist       *   in_quad ;
00639     cpl_imagelist       **  results ;
00640     cpl_imagelist       *   res_quad[4] ;
00641     cpl_image           *   err_quad[4] ;
00642     cpl_image           *   badres_mask[4] ;
00643     cpl_image           *   flat_image ;
00644     cpl_image           *   dark ;
00645     cpl_propertylist    *   plist;
00646     double                  gradient ;
00647     double                  flat_dit;
00648     cpl_image           *   bpmflat;
00649     int                     j, k ;
00650     int                     idet;
00651 
00652     /* Test entries */
00653     if (flatframes == NULL) return NULL ;
00654 
00655     /* Initialise */
00656     nima = cpl_frameset_get_size(flatframes) ;
00657     if (nima < 3) {
00658         cpl_msg_error(__func__, "Not enough frames (%d)", nima) ;
00659         return NULL ;
00660     }
00661     for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++)
00662         hawki_cal_flat_outputs.norm[idet] = 1.0 ;
00663     
00664     /* Compute statistics */
00665     cpl_msg_info(__func__, "Compute statistics") ;
00666     cpl_msg_indent_more() ;
00667     /* Loop on the HAWKI_NB_DETECTORS chips */
00668     for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++) 
00669     {
00670 
00671         /* Compute some stats on input images */
00672         cpl_msg_info(__func__, "Chip number %d", idet+1) ;
00673         cpl_msg_info(__func__, "image      min        max        med     rms") ;
00674         cpl_msg_info(__func__, "--------------------------------------------") ;
00675         medians[idet] = cpl_vector_new(nima);
00676         stdevs[idet] = cpl_vector_new(nima);
00677         for (j=0 ; j<nima ; j++)
00678         {
00679             /* Load the image */
00680             ima_cur = hawki_load_image(flatframes, j, idet+1, CPL_TYPE_FLOAT) ;
00681 
00682             /* Compute the stats */
00683             if(hawki_image_stats_fill_from_image
00684                 (raw_flat_stats,
00685                  ima_cur,
00686                  hawki_cal_flat_config.llx,
00687                  hawki_cal_flat_config.lly,
00688                  hawki_cal_flat_config.urx,
00689                  hawki_cal_flat_config.ury,
00690                  idet,
00691                  j) !=0 )
00692             {
00693                 cpl_msg_error(__func__, "Cannot compute stats on image %d",j+1);
00694                 cpl_msg_indent_less() ;
00695                 cpl_image_delete(ima_cur);
00696                 for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
00697                 for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
00698                 return NULL ;
00699             }
00700             
00701             if(hawki_cal_flat_config.extra_stats)
00702             {
00703                 if(hawki_image_stats_odd_even_column_row_fill_from_image
00704                         (raw_flat_odd_column_stats,
00705                          raw_flat_even_column_stats,
00706                          raw_flat_odd_row_stats,
00707                          raw_flat_even_row_stats,
00708                          ima_cur,
00709                          idet,
00710                          j) !=0 )
00711                 {
00712                     cpl_msg_error(__func__, "Cannot compute extra stats");
00713                     cpl_msg_indent_less() ;
00714                     cpl_image_delete(ima_cur);
00715                     for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
00716                     for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
00717                     return NULL ;
00718                 }
00719             }
00720             cpl_vector_set(medians[idet], j, cpl_table_get_double
00721                            (raw_flat_stats[idet],HAWKI_COL_STAT_MED,j,NULL )) ;
00722             cpl_vector_set(stdevs[idet], j, cpl_table_get_double
00723                            (raw_flat_stats[idet],HAWKI_COL_STAT_RMS,j,NULL )) ;
00724             cpl_msg_info(__func__, "%02d   %10.2f %10.2f %10.2f %10.2f",
00725                     j+1,
00726                     cpl_table_get_double(raw_flat_stats[idet],
00727                                          HAWKI_COL_STAT_MIN,j,NULL ),
00728                     cpl_table_get_double(raw_flat_stats[idet],
00729                                          HAWKI_COL_STAT_MAX,j,NULL ),
00730                     cpl_table_get_double(raw_flat_stats[idet],
00731                                          HAWKI_COL_STAT_MED,j,NULL ),
00732                     cpl_table_get_double(raw_flat_stats[idet],
00733                                          HAWKI_COL_STAT_RMS,j,NULL ));
00734             if (cpl_table_get_double
00735                     (raw_flat_stats[idet],HAWKI_COL_STAT_MED,j,NULL ) < 1e-6) 
00736             {
00737                 cpl_msg_error(__func__, "image %d has negative flux: aborting", 
00738                         j+1) ;
00739                 cpl_msg_indent_less() ;
00740                 for (k=0 ; k<=idet ; k++) cpl_vector_delete(medians[k]) ;
00741                 for (k=0 ; k<=idet ; k++) cpl_vector_delete(stdevs[k]) ;
00742                 return NULL ;
00743             }
00744             cpl_image_delete(ima_cur);
00745         }
00746         cpl_msg_info(__func__, "--------------------------------------------") ;
00747 
00748         /* Compute min max stdev and mean of the medians */
00749         hawki_cal_flat_outputs.med_min[idet]   = 
00750             cpl_vector_get_min(medians[idet]);
00751         hawki_cal_flat_outputs.med_max[idet]   = 
00752             cpl_vector_get_max(medians[idet]);
00753         hawki_cal_flat_outputs.med_avg[idet]   = 
00754             cpl_vector_get_mean(medians[idet]);
00755         hawki_cal_flat_outputs.med_med[idet]   =
00756             cpl_vector_get_median_const(medians[idet]);
00757         hawki_cal_flat_outputs.med_stdev[idet] = 
00758             cpl_vector_get_stdev(medians[idet]);
00759         
00760         /* See if flux gradient is large enough for a correct fit */
00761         gradient=fabs(hawki_cal_flat_outputs.med_max[idet]/ 
00762                 hawki_cal_flat_outputs.med_min[idet]) ;
00763         if (gradient < 4.0) {
00764             /* cpl_msg_warning(__func__, "Low flux gradient: %g", gradient) ;*/
00765         }
00766     }
00767     cpl_msg_indent_less() ;
00768     
00769     /* Allocate for results */
00770     results = cpl_malloc(4 * sizeof(cpl_imagelist*)) ;
00771     results[0] = cpl_imagelist_new() ;
00772     results[1] = cpl_imagelist_new() ;
00773     results[2] = cpl_imagelist_new() ;
00774     if (hawki_cal_flat_config.second_pass)  results[3] = cpl_imagelist_new() ; 
00775     else                                    results[3] = NULL ;
00776 
00777     cpl_msg_info(__func__, "Compute the flat") ;
00778     cpl_msg_indent_more() ;
00779     /* Loop on the HAWKI_NB_DETECTORS chips */
00780     for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++) {
00781         cpl_msg_info(__func__, "Chip number %d", idet+1) ;
00782         cpl_msg_indent_more() ;
00783        
00784         /* Frames selection */
00785         cpl_msg_info(__func__, "Apply the frames selection");
00786         if ((hawki_cal_flat_select(medians[idet], 
00787                                    stdevs[idet],
00788                                    hawki_cal_flat_config.select_auto,
00789                                    hawki_cal_flat_config.select_auto_max_bins,
00790                                    hawki_cal_flat_config.select_min_level[idet],
00791                                    hawki_cal_flat_config.select_max_level[idet],
00792                                    hawki_cal_flat_config.select_max_rms[idet],
00793                                    hawki_cal_flat_config.select_min_nframes,
00794                                    selected[idet])) == -1)
00795         {
00796             cpl_msg_error(__func__, "Cannot apply the frames selection") ;
00797             cpl_imagelist_delete(results[0]) ;
00798             cpl_imagelist_delete(results[1]) ;
00799             cpl_imagelist_delete(results[2]) ;
00800             if (hawki_cal_flat_config.second_pass) 
00801                 cpl_imagelist_delete(results[3]) ;
00802             cpl_free(results) ;
00803             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00804                 cpl_vector_delete(medians[k]) ;
00805             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00806                 cpl_vector_delete(stdevs[k]) ;
00807             return NULL ;
00808         }
00809 
00810         /* Apply the medians subselection */
00811         sub_medians = hawki_cal_flat_extract_vector
00812             (medians[idet], selected[idet]) ;
00813 
00814         /* Loop on the 4 quadrants */
00815         for (j=0 ; j<4 ; j++) {
00816             /* Load input image chips */
00817             if ((in_quad = hawki_load_quadrants(flatframes, idet+1, j+1, 
00818                             CPL_TYPE_FLOAT))==NULL) {
00819                 cpl_msg_error(__func__, "Cannot load the raw quadrants") ;
00820                 cpl_imagelist_delete(results[0]) ;
00821                 cpl_imagelist_delete(results[1]) ;
00822                 cpl_imagelist_delete(results[2]) ;
00823                 if (hawki_cal_flat_config.second_pass) 
00824                     cpl_imagelist_delete(results[3]) ;
00825                 cpl_free(results) ;
00826                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00827                     cpl_vector_delete(medians[k]) ;
00828                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00829                     cpl_vector_delete(stdevs[k]) ;
00830                 cpl_vector_delete(sub_medians) ;
00831                 return NULL ;
00832             }
00833        
00834             /* Apply subselection of the frames */
00835             cpl_imagelist_erase(in_quad, selected[idet]);
00836 
00837             /* Apply dark correction to all planes if requested */
00838             if (dark_file) {
00839                 if (j==0) cpl_msg_info(__func__, "Subtracting dark") ;
00840                 /* Load dark */
00841                 if ((dark = hawki_load_quadrant_from_file(dark_file,
00842                                 idet+1, j+1, CPL_TYPE_FLOAT)) == NULL) {
00843                     cpl_msg_error(__func__, "Cannot load the dark quadrants") ;
00844                     cpl_imagelist_delete(in_quad) ;
00845                     cpl_imagelist_delete(results[0]) ;
00846                     cpl_imagelist_delete(results[1]) ;
00847                     cpl_imagelist_delete(results[2]) ;
00848                     if (hawki_cal_flat_config.second_pass) 
00849                         cpl_imagelist_delete(results[3]) ;
00850                     cpl_free(results) ;
00851                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00852                         cpl_vector_delete(medians[k]) ;
00853                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00854                         cpl_vector_delete(stdevs[k]) ;
00855                     cpl_vector_delete(sub_medians) ;
00856                     return NULL ;
00857                 }
00858 
00859                 /* Multiply by the dit */
00860                 if ((plist=cpl_propertylist_load
00861                         (cpl_frame_get_filename
00862                          (cpl_frameset_get_first_const(flatframes)), 0)) == NULL) 
00863                 {
00864                     cpl_msg_error(__func__, "Cannot get header from frame");
00865                     cpl_imagelist_delete(in_quad) ;
00866                     cpl_imagelist_delete(results[0]) ;
00867                     cpl_imagelist_delete(results[1]) ;
00868                     cpl_imagelist_delete(results[2]) ;
00869                     if (hawki_cal_flat_config.second_pass) 
00870                         cpl_imagelist_delete(results[3]) ;
00871                     cpl_free(results) ;
00872                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00873                         cpl_vector_delete(medians[k]) ;
00874                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00875                         cpl_vector_delete(stdevs[k]) ;
00876                     cpl_vector_delete(sub_medians) ;
00877                     cpl_image_delete(dark);
00878                     return NULL ;
00879                 }
00880                 flat_dit = hawki_pfits_get_dit(plist);
00881                 cpl_image_multiply_scalar(dark, flat_dit);
00882                 cpl_propertylist_delete(plist);
00883 
00884                 /* Dark correction */
00885                 cpl_imagelist_subtract_image(in_quad, dark) ;
00886                 cpl_image_delete(dark) ;
00887                 if (cpl_error_get_code() != CPL_ERROR_NONE) {
00888                     cpl_msg_warning(__func__,"Cannot subtract the dark frame");
00889                     cpl_error_reset() ;
00890                 }
00891             }
00892         
00893             /* Fit slopes */
00894             err_quad[j] = cpl_image_duplicate(cpl_imagelist_get(in_quad, 0)) ;
00895             res_quad[j] = cpl_fit_imagelist_polynomial(sub_medians, in_quad, 
00896                     0, 1, CPL_FALSE, CPL_TYPE_FLOAT, err_quad[j]) ;
00897             if (res_quad[j] == NULL) {
00898                 cpl_msg_error(__func__, "Cannot create twilight flat-field") ;
00899                 cpl_imagelist_delete(results[0]) ;
00900                 cpl_imagelist_delete(results[1]) ;
00901                 cpl_imagelist_delete(results[2]) ;
00902                 if (hawki_cal_flat_config.second_pass) 
00903                     cpl_imagelist_delete(results[3]) ;
00904                 cpl_free(results) ;
00905                 cpl_imagelist_delete(in_quad) ;
00906                 for (k=0 ; k<j ; k++) cpl_imagelist_delete(res_quad[k]) ;
00907                 for (k=0 ; k<=j ; k++) cpl_image_delete(err_quad[k]) ;
00908                 for (k=0 ; k<j ; k++) 
00909                     if (badres_mask[k]) cpl_image_delete(badres_mask[k]) ;
00910                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00911                     cpl_vector_delete(medians[k]) ;
00912                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00913                     cpl_vector_delete(stdevs[k]) ;
00914                 cpl_vector_delete(sub_medians) ;
00915                 return NULL ;
00916             }
00917 
00918             /* Handle the pixels with a high error */
00919             badres_mask[j] = NULL ;
00920             if (hawki_cal_flat_config.second_pass) {
00921                 if (j==0) cpl_msg_info(__func__, 
00922                         "Second pass to clean the outliers") ;
00923                 if (hawki_cal_flat_clean_outliers(err_quad[j], res_quad[j],
00924                             in_quad, sub_medians, &(badres_mask[j])) == -1) {
00925                     cpl_msg_error(__func__, "Cannot clean the outliers") ;
00926                     cpl_imagelist_delete(results[0]) ;
00927                     cpl_imagelist_delete(results[1]) ;
00928                     cpl_imagelist_delete(results[2]) ;
00929                     cpl_imagelist_delete(results[3]) ;
00930                     cpl_free(results) ;
00931                     cpl_imagelist_delete(in_quad) ;
00932                     for (k=0 ; k<=j ; k++) cpl_imagelist_delete(res_quad[k]) ;
00933                     for (k=0 ; k<=j ; k++) cpl_image_delete(err_quad[k]) ;
00934                     for (k=0 ; k<=j ; k++) cpl_image_delete(badres_mask[k]) ;
00935                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00936                         cpl_vector_delete(medians[k]) ;
00937                     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00938                         cpl_vector_delete(stdevs[k]) ;
00939                     cpl_vector_delete(sub_medians) ;
00940                     cpl_msg_indent_less() ;
00941                     return NULL ;
00942                 }
00943             }
00944             cpl_imagelist_delete(in_quad) ;
00945         }
00946         cpl_vector_delete(sub_medians) ;
00947             
00948         /* Rebuild the quadrants and put in results */
00949         /* Rebuild RESULTS */
00950         big_ima = hawki_rebuild_quadrants(
00951                 cpl_imagelist_get(res_quad[0],1), 
00952                 cpl_imagelist_get(res_quad[1],1), 
00953                 cpl_imagelist_get(res_quad[2],1), 
00954                 cpl_imagelist_get(res_quad[3],1)) ;
00955         for (j=0 ; j<4 ; j++) cpl_imagelist_delete(res_quad[j]) ;
00956         cpl_imagelist_set(results[0], big_ima, idet) ;
00957         if (big_ima == NULL) {
00958             cpl_msg_error(__func__, "Cannot rebuild the image") ;
00959             cpl_imagelist_delete(results[0]) ;
00960             cpl_imagelist_delete(results[1]) ;
00961             cpl_imagelist_delete(results[2]) ;
00962             if (hawki_cal_flat_config.second_pass) 
00963                 cpl_imagelist_delete(results[3]) ;
00964             cpl_free(results) ;
00965             for (j=0 ; j<4 ; j++) cpl_image_delete(err_quad[j]) ;
00966             for (j=0 ; j<4 ; j++) 
00967                 if (badres_mask[j]) cpl_image_delete(badres_mask[j]) ;
00968             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00969                 cpl_vector_delete(medians[k]) ;
00970             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00971                 cpl_vector_delete(stdevs[k]) ;
00972             return NULL ;
00973         }
00974 
00975         /* Rebuild ERROR */
00976         big_ima = hawki_rebuild_quadrants(err_quad[0], err_quad[1], 
00977                 err_quad[2], err_quad[3]) ;
00978         for (j=0 ; j<4 ; j++) cpl_image_delete(err_quad[j]) ;
00979         if (big_ima == NULL) {
00980             cpl_msg_error(__func__, "Cannot rebuild the image") ;
00981             cpl_imagelist_delete(results[0]) ;
00982             cpl_imagelist_delete(results[1]) ;
00983             cpl_imagelist_delete(results[2]) ;
00984             if (hawki_cal_flat_config.second_pass) 
00985                 cpl_imagelist_delete(results[3]) ;
00986             cpl_free(results) ;
00987             for (j=0 ; j<4 ; j++) 
00988                 if (badres_mask[j]) cpl_image_delete(badres_mask[j]) ;
00989             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00990                 cpl_vector_delete(medians[k]) ;
00991             for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
00992                 cpl_vector_delete(stdevs[k]) ;
00993             return NULL ;
00994         }
00995         cpl_imagelist_set(results[1], big_ima, idet) ;
00996        
00997         /* Rebuild BADRES_MASK */
00998         big_badres = hawki_rebuild_quadrants(badres_mask[0], badres_mask[1], 
00999                 badres_mask[2], badres_mask[3]) ;
01000         if (hawki_cal_flat_config.second_pass) {
01001             for (j=0 ; j<4 ; j++) cpl_image_delete(badres_mask[j]) ;
01002             cpl_imagelist_set(results[3], big_badres, idet) ;
01003         }
01004 
01005         if (hawki_cal_flat_config.normalise) {
01006             /* Normalize gain */
01007             cpl_msg_info(__func__, "Normalise the flat") ;
01008             flat_image = cpl_imagelist_get(results[0], idet) ;
01009             hawki_cal_flat_outputs.norm[idet] = 
01010                 cpl_image_get_median(flat_image) ;
01011             cpl_image_divide_scalar
01012                 (flat_image, hawki_cal_flat_outputs.norm[idet]);
01013             if (cpl_error_get_code()) {
01014                 cpl_msg_error(__func__, "Cannot normalise") ;
01015                 cpl_imagelist_delete(results[0]) ;
01016                 cpl_imagelist_delete(results[1]) ;
01017                 cpl_imagelist_delete(results[2]) ;
01018                 if (hawki_cal_flat_config.second_pass) 
01019                     cpl_imagelist_delete(results[3]) ;
01020                 cpl_free(results) ;
01021                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
01022                     cpl_vector_delete(medians[k]) ;
01023                 for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) 
01024                     cpl_vector_delete(stdevs[k]) ;
01025                 return NULL ;
01026             }
01027         }
01028 
01029         /* BPM from the flat */
01030         cpl_msg_info(__func__, "Compute the BPM from the flat") ;
01031         bpmflat=hawki_compute_flatbpm(cpl_imagelist_get(results[0],idet),
01032                 hawki_cal_flat_config.sigma_bpm,
01033                 hawki_cal_flat_config.lowval_bpm, 
01034                 hawki_cal_flat_config.highval_bpm);
01035         
01036         cpl_imagelist_set(results[2], bpmflat, idet) ;
01037         hawki_cal_flat_outputs.nb_badpix[idet]=
01038             (int)cpl_image_get_flux(bpmflat);
01039 
01040         cpl_msg_indent_less() ;
01041     }
01042     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) cpl_vector_delete(medians[k]) ;
01043     for (k=0 ; k<HAWKI_NB_DETECTORS ; k++) cpl_vector_delete(stdevs[k]) ;
01044     cpl_msg_indent_less() ;
01045 
01046     return results ;
01047 }
01048 
01049 /*----------------------------------------------------------------------------*/
01059 /*----------------------------------------------------------------------------*/
01060 static int hawki_cal_flat_clean_outliers(
01061         cpl_image           *   error,
01062         cpl_imagelist       *   fit_res,
01063         cpl_imagelist       *   raw,
01064         cpl_vector          *   medians,
01065         cpl_image           **  recomp_mask)
01066 {
01067     cpl_mask        *   recompute ;
01068     cpl_binary      *   precompute ;
01069     double              med, stdev, threshold1, threshold2 ;
01070     int                 nx, ny, pos, nbad, nima, out, ind, pix ;
01071     cpl_image       *   cur_ima ;
01072     float           *   pimaf ;
01073     double              val, fit_val, a, b, max ;
01074     cpl_vector      *   z_pix ;
01075     double          *   pz_pix ;
01076     cpl_image       *   onepix ;
01077     cpl_vector      *   med_purged ;
01078     cpl_imagelist   *   raw_purged ;
01079     cpl_imagelist   *   fit_one ;
01080     cpl_image       *   err_one ;
01081     int                 i, j, k ;
01082 
01083     /* Check entries */
01084     if (error == NULL) return -1 ;
01085     if (fit_res == NULL) return -1 ;
01086     if (raw == NULL) return -1 ;
01087 
01088     /* Initialise */
01089     if (recomp_mask) *recomp_mask = NULL ;
01090 
01091     /* Identify the places to recompute */
01092     med = cpl_image_get_median_dev(error, &stdev) ;
01093     threshold1 = med - hawki_cal_flat_config.sigma_badres * stdev ;
01094     threshold2 = med + hawki_cal_flat_config.sigma_badres * stdev ;
01095     recompute = cpl_mask_threshold_image_create(error,threshold1,threshold2) ;
01096     cpl_mask_not(recompute) ;
01097 
01098     if ((nbad=cpl_mask_count(recompute)) == 0) {
01099         if (recomp_mask)    
01100             *recomp_mask = cpl_image_new_from_mask(recompute) ;
01101         cpl_mask_delete(recompute) ;
01102         return 0 ;
01103     }
01104     nx = cpl_image_get_size_x(error) ;
01105     ny = cpl_image_get_size_y(error) ;
01106     nima = cpl_imagelist_get_size(raw) ;
01107 
01108     /* Get access to the mask */
01109     precompute = cpl_mask_get_data(recompute) ;
01110     for (j=0 ; j<ny ; j++) {
01111         for (i=0 ; i<nx ; i++) {
01112             pos = i + j*nx ;
01113             /* The pixel has to be recomputed */
01114             if (precompute[pos] == CPL_BINARY_1) {
01115                 /* Get the pix_val-fit in a vector */
01116                 z_pix = cpl_vector_new(nima) ;
01117                 for (k=0 ; k<nima ; k++) {
01118                     cur_ima = cpl_imagelist_get(fit_res, 0) ;
01119                     pimaf = cpl_image_get_data_float(cur_ima) ;
01120                     a = pimaf[pos] ;
01121                     cur_ima = cpl_imagelist_get(fit_res, 1) ;
01122                     pimaf = cpl_image_get_data_float(cur_ima) ;
01123                     b = pimaf[pos] ;
01124                     med = cpl_vector_get(medians, k) ;
01125                     fit_val = a + b * med ;
01126                     cur_ima = cpl_imagelist_get(raw, k) ;
01127                     pimaf = cpl_image_get_data_float(cur_ima) ;
01128                     cpl_vector_set(z_pix, k, (double)(pimaf[pos]-fit_val)) ;
01129                 }
01130 
01131                 /* Identify the outlier */
01132                 out = -1 ;
01133                 max = -1.0 ;
01134                 pz_pix = cpl_vector_get_data(z_pix) ;
01135                 for (k=0 ; k<nima ; k++) {
01136                     if (fabs(pz_pix[k]) > max) {
01137                         max = fabs(pz_pix[k]) ;
01138                         out = k ;
01139                     }
01140                 }
01141                 cpl_vector_delete(z_pix) ;
01142 
01143                 /* Create 1-pixel purged image list and the purged medians */
01144                 med_purged = cpl_vector_new(nima-1) ;
01145                 raw_purged = cpl_imagelist_new() ;
01146                 ind = 0 ;
01147                 for (k=0 ; k<nima ; k++) {
01148                     if (k != out) {
01149                         /* Fill raw_purged */
01150                         cur_ima = cpl_imagelist_get(raw, k) ;
01151                         onepix=cpl_image_extract(cur_ima, i+1, j+1, i+1, j+1) ;
01152                         cpl_imagelist_set(raw_purged, onepix, ind) ;
01153                         /* Fill med_purged */
01154                         cpl_vector_set(med_purged, ind,
01155                                 cpl_vector_get(medians, k)) ;
01156                         ind ++;
01157                     }
01158                 }
01159 
01160                 /* Perform the fit */
01161                 err_one = cpl_image_duplicate(cpl_imagelist_get(raw_purged,0));
01162                 fit_one = cpl_fit_imagelist_polynomial(med_purged, raw_purged, 
01163                         0, 1, CPL_FALSE, CPL_TYPE_FLOAT, err_one) ;
01164                 if (fit_one == NULL) {
01165                     cpl_msg_error(__func__, "Cannot fit in second pass") ;
01166                     cpl_mask_delete(recompute) ;
01167                     cpl_vector_delete(med_purged) ;
01168                     cpl_imagelist_delete(raw_purged) ;
01169                     cpl_image_delete(err_one) ;
01170                     return -1 ;
01171                 }
01172                 cpl_vector_delete(med_purged) ;
01173                 cpl_imagelist_delete(raw_purged) ;
01174 
01175                 /* Write the result in the input */
01176                 val = cpl_image_get(err_one, 1, 1, &pix) ;
01177                 cpl_image_set(error, i+1, j+1, val) ;
01178                 cpl_image_delete(err_one) ;
01179 
01180                 cur_ima = cpl_imagelist_get(fit_one, 0) ;
01181                 val = cpl_image_get(cur_ima, 1, 1, &pix) ;
01182                 cur_ima = cpl_imagelist_get(fit_res, 0) ;
01183                 cpl_image_set(cur_ima, i+1, j+1, val) ;
01184               
01185                 cur_ima = cpl_imagelist_get(fit_one, 1) ;
01186                 val = cpl_image_get(cur_ima, 1, 1, &pix) ;
01187                 cur_ima = cpl_imagelist_get(fit_res, 1) ;
01188                 cpl_image_set(cur_ima, i+1, j+1, val) ;
01189                 cpl_imagelist_delete(fit_one) ;
01190             }
01191         }
01192     }
01193     if (recomp_mask)    
01194         *recomp_mask = cpl_image_new_from_mask(recompute) ;
01195     cpl_mask_delete(recompute) ;
01196 
01197     /* Return  */
01198     if (cpl_error_get_code()) return -1 ;
01199     else return 0 ;
01200 }
01201 
01202 /*----------------------------------------------------------------------------*/
01214 /*----------------------------------------------------------------------------*/
01215 static int hawki_cal_flat_save
01216 (cpl_imagelist     ** flat,
01217  cpl_table         ** raw_flat_stats,
01218  cpl_table         ** raw_flat_odd_column_stats,
01219  cpl_table         ** raw_flat_even_column_stats,
01220  cpl_table         ** raw_flat_odd_row_stats,
01221  cpl_table         ** raw_flat_even_row_stats,
01222  cpl_vector        ** raw_selected,
01223  int                  set_nb,
01224  const cpl_frame   *  bpmdark,
01225  cpl_frameset      *  flat_frames,
01226  cpl_frameset      *  calib_frames,
01227  cpl_parameterlist *  parlist,
01228  cpl_frameset      *  set_tot)
01229 {
01230     cpl_propertylist    **  qclists ;
01231     cpl_imagelist       *   bpm ;
01232     const cpl_frame     *   ref_frame ;
01233     cpl_frameset        *   used_frames;
01234     char                *   filename ;
01235     cpl_propertylist    *   inputlist ;
01236     int                     ext_nb ;
01237     const char          *   recipe_name = "hawki_cal_flat" ;
01238     int                     i ;
01239     int                     iflat;
01240     int                     nflat;
01241     int                     nused;
01242     char                    key_name[72];
01243 
01244     /* Get the reference frame */
01245     ref_frame = irplib_frameset_get_first_from_group(flat_frames, CPL_FRAME_GROUP_RAW) ;
01246 
01247     /* Create the QC lists */
01248     qclists = cpl_malloc(HAWKI_NB_DETECTORS * sizeof(cpl_propertylist*)) ;
01249     for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
01250         qclists[i] = cpl_propertylist_new() ;
01251         
01252         /* Add the raw flat selection keywords */
01253         nflat = cpl_vector_get_size(raw_selected[i]);
01254         nused = 0;
01255         for(iflat = 0; iflat < nflat; ++iflat)
01256         {
01257             snprintf(key_name, 72, "ESO QC RAW%02d USED", iflat + 1);
01258             cpl_propertylist_append_bool
01259                 (qclists[i], key_name, 
01260                 (cpl_vector_get(raw_selected[i], iflat) + 1) / 2);
01261             cpl_table_set_int
01262                 (raw_flat_stats[i],HAWKI_COL_STAT_USED, iflat,
01263                  cpl_vector_get(raw_selected[i], iflat));
01264             if(hawki_cal_flat_config.extra_stats)
01265             {
01266                 cpl_table_set_int
01267                     (raw_flat_odd_column_stats[i],HAWKI_COL_STAT_USED, iflat,
01268                      cpl_vector_get(raw_selected[i], iflat));
01269                 cpl_table_set_int
01270                     (raw_flat_even_column_stats[i],HAWKI_COL_STAT_USED, iflat,
01271                      cpl_vector_get(raw_selected[i], iflat));
01272                 cpl_table_set_int
01273                     (raw_flat_odd_row_stats[i],HAWKI_COL_STAT_USED, iflat,
01274                      cpl_vector_get(raw_selected[i], iflat));
01275                 cpl_table_set_int
01276                     (raw_flat_even_row_stats[i],HAWKI_COL_STAT_USED, iflat,
01277                      cpl_vector_get(raw_selected[i], iflat));
01278             }
01279             if(cpl_vector_get(raw_selected[i], iflat) == 1)
01280                 nused++;
01281         }
01282         
01283         /* Add the master flat statistics keywords */
01284         cpl_propertylist_append_int(qclists[i], "ESO QC FLAT NBADPIX",
01285                                     hawki_cal_flat_outputs.nb_badpix[i]);
01286         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT NORM",
01287                                        hawki_cal_flat_outputs.norm[i]) ;
01288         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMEAN",
01289                                        hawki_cal_flat_outputs.med_avg[i]) ;
01290         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMED",
01291                                        hawki_cal_flat_outputs.med_med[i]) ;
01292         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDSTDEV",
01293                                        hawki_cal_flat_outputs.med_stdev[i]) ;
01294         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMIN",
01295                                        hawki_cal_flat_outputs.med_min[i]) ;
01296         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDMAX",
01297                                        hawki_cal_flat_outputs.med_max[i]) ;
01298         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDRANG",
01299                                        hawki_cal_flat_outputs.med_max[i] -
01300                                        hawki_cal_flat_outputs.med_min[i]);
01301         cpl_propertylist_append_double(qclists[i], "ESO QC FLAT MEDNRANG",
01302                                        (hawki_cal_flat_outputs.med_max[i] -
01303                                         hawki_cal_flat_outputs.med_min[i]) /
01304                                         hawki_cal_flat_outputs.med_med[i]);
01305         cpl_propertylist_append_int(qclists[i], "ESO QC DATANCOM",
01306                                     nused) ;
01307 
01308         /* Propagate some keywords from input raw frame extensions */
01309         ext_nb = hawki_get_ext_from_detector(
01310                 cpl_frame_get_filename(ref_frame), i+1);
01311         inputlist = cpl_propertylist_load_regexp(
01312                 cpl_frame_get_filename(ref_frame), ext_nb,
01313                 HAWKI_HEADER_EXT_FORWARD, 0) ;
01314         cpl_propertylist_append(qclists[i], inputlist) ;
01315 
01316         /* Cleaning */
01317         cpl_propertylist_delete(inputlist) ;
01318     }
01319     /* Statistics of the raw images in the QC */
01320     hawki_image_stats_stats(raw_flat_stats, qclists);
01321 
01322     /* Get the used frames */
01323     used_frames = cpl_frameset_duplicate(flat_frames);
01324     for(i = 0; i< cpl_frameset_get_size(calib_frames); ++i)
01325         cpl_frameset_insert(used_frames, 
01326                 cpl_frame_duplicate(cpl_frameset_get_frame(calib_frames, i)));
01327 
01328     /* Write the flat image */
01329     filename = cpl_sprintf("hawki_cal_flat_set%02d.fits", set_nb) ;
01330     hawki_imagelist_save(set_tot,
01331                          parlist,
01332                          used_frames,
01333                          flat[0],
01334                          recipe_name,
01335                          HAWKI_CALPRO_FLAT,
01336                          HAWKI_PROTYPE_FLAT, 
01337                          NULL,
01338                          (const cpl_propertylist**)qclists,
01339                          filename) ;
01340     cpl_free(filename) ;
01341     
01342     /* Write the error map */
01343     filename = cpl_sprintf("hawki_cal_flat_err_set%02d.fits", set_nb) ;
01344     hawki_imagelist_save(set_tot,
01345                          parlist,
01346                          used_frames,
01347                          flat[1],
01348                          recipe_name,
01349                          HAWKI_CALPRO_FLAT_ERRMAP,
01350                          HAWKI_PROTYPE_ERRMAP, 
01351                          NULL,
01352                          (const cpl_propertylist**)qclists,
01353                          filename) ;
01354     cpl_free(filename) ;
01355  
01356     /* Write the Cold pixels map */
01357     filename = cpl_sprintf("hawki_cal_flat_bpmflat_set%02d.fits", set_nb) ;
01358     hawki_imagelist_save(set_tot,
01359                          parlist,
01360                          used_frames,
01361                          flat[2],
01362                          recipe_name,
01363                          HAWKI_CALPRO_BPM_COLD,
01364                          HAWKI_PROTYPE_BPM,
01365                          NULL,
01366                          (const cpl_propertylist**)qclists,
01367                          filename) ;
01368     cpl_free(filename) ;
01369 
01370     if (flat[3] != NULL) {
01371         /* Write the recomputed map */
01372         filename=cpl_sprintf("hawki_cal_flat_recomputed_set%02d.fits", set_nb) ;
01373         hawki_imagelist_save(set_tot,
01374                              parlist,
01375                              used_frames,
01376                              flat[3],
01377                              recipe_name,
01378                              HAWKI_CALPRO_FLAT_RECOMPUTED,
01379                              HAWKI_PROTYPE_FLAT,
01380                              NULL,
01381                              (const cpl_propertylist**)qclists,
01382                              filename) ;
01383         cpl_free(filename) ;
01384     }
01385 
01386     /* If the HOT pixel map is passed, merge with the COLD one */
01387     if (bpmdark != NULL) {
01388         if ((bpm = hawki_cal_flat_merge_bpms(bpmdark, flat[2])) == NULL) {
01389             cpl_msg_error(__func__, "Cannot merge bad pixel maps") ;
01390         } else {
01391             filename=cpl_sprintf("hawki_cal_flat_bpm_set%02d.fits", set_nb) ;
01392             /* Get the used frames for statistics */
01393             hawki_imagelist_save(set_tot,
01394                                  parlist,
01395                                  used_frames,
01396                                  bpm,
01397                                  recipe_name,
01398                                  HAWKI_CALPRO_BPM,
01399                                  HAWKI_PROTYPE_BPM,
01400                                  NULL,
01401                                  (const cpl_propertylist**)qclists,
01402                                  filename) ;
01403             cpl_free(filename) ;
01404             cpl_imagelist_delete(bpm) ;
01405         }
01406     }
01407     cpl_frameset_delete(used_frames);
01408 
01409     /* Get the used frames for statistics */
01410     used_frames = cpl_frameset_duplicate(flat_frames);
01411     
01412     /* Write the table with the statistics */
01413     filename = cpl_sprintf("hawki_cal_flat_stats_set%02d.fits", set_nb) ;
01414     hawki_tables_save(set_tot,
01415                       parlist,
01416                       used_frames,
01417                       (const cpl_table **)raw_flat_stats,
01418                       recipe_name,
01419                       HAWKI_CALPRO_FLAT_STATS,
01420                       HAWKI_PROTYPE_FLAT_STATS,
01421                       NULL,
01422                       (const cpl_propertylist **)qclists,
01423                       filename) ;
01424     cpl_free(filename) ;
01425     
01426     if(hawki_cal_flat_config.extra_stats)
01427     {
01428         filename = cpl_sprintf("hawki_cal_flat_stats_ec_set%02d.fits", set_nb);
01429         hawki_tables_save(set_tot,
01430                           parlist,
01431                           used_frames,
01432                           (const cpl_table **)raw_flat_even_column_stats,
01433                           recipe_name,
01434                           HAWKI_CALPRO_FLAT_STATS_EVEN_COL,
01435                           HAWKI_PROTYPE_FLAT_STATS_EVEN_COL,
01436                           NULL,
01437                           (const cpl_propertylist **)qclists,
01438                           filename) ;
01439         cpl_free(filename) ;
01440         filename = cpl_sprintf("hawki_cal_flat_stats_oc_set%02d.fits", set_nb);
01441         hawki_tables_save(set_tot,
01442                           parlist,
01443                           used_frames,
01444                           (const cpl_table **)raw_flat_odd_column_stats,
01445                           recipe_name,
01446                           HAWKI_CALPRO_FLAT_STATS_ODD_COL,
01447                           HAWKI_PROTYPE_FLAT_STATS_ODD_COL,
01448                           NULL,
01449                           (const cpl_propertylist **)qclists,
01450                           filename) ;
01451         cpl_free(filename) ;
01452         filename = cpl_sprintf("hawki_cal_flat_stats_er_set%02d.fits", set_nb);
01453         hawki_tables_save(set_tot,
01454                           parlist,
01455                           used_frames,
01456                           (const cpl_table **)raw_flat_even_row_stats,
01457                           recipe_name,
01458                           HAWKI_CALPRO_FLAT_STATS_EVEN_ROW,
01459                           HAWKI_PROTYPE_FLAT_STATS_EVEN_ROW,
01460                           NULL,
01461                           (const cpl_propertylist **)qclists,
01462                           filename) ;
01463         cpl_free(filename) ;
01464         filename = cpl_sprintf("hawki_cal_flat_stats_or_set%02d.fits", set_nb);
01465         hawki_tables_save(set_tot,
01466                           parlist,
01467                           used_frames,
01468                           (const cpl_table **)raw_flat_odd_row_stats,
01469                           recipe_name,
01470                           HAWKI_CALPRO_FLAT_STATS_ODD_ROW,
01471                           HAWKI_PROTYPE_FLAT_STATS_ODD_ROW,
01472                           NULL,
01473                           (const cpl_propertylist **)qclists,
01474                           filename) ;
01475         cpl_free(filename) ;
01476     }
01477     cpl_frameset_delete(used_frames);
01478         
01479     /* Free and return */
01480     for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
01481         cpl_propertylist_delete(qclists[i]) ;
01482     }
01483     cpl_free(qclists) ;
01484     if (cpl_error_get_code()) return -1 ;
01485     return  0;
01486 }
01487 
01488 /*----------------------------------------------------------------------------*/
01495 /*----------------------------------------------------------------------------*/
01496 static int hawki_cal_flat_compare(
01497         const cpl_frame *   frame1,
01498         const cpl_frame *   frame2)
01499 {
01500     int                     comparison ;
01501     cpl_propertylist    *   plist1 ;
01502     cpl_propertylist    *   plist2 ;
01503     const char          *   sval1,
01504                         *   sval2 ;
01505     double                  dval1, dval2 ;
01506 
01507     
01508     /* Test entries */
01509     if (frame1==NULL || frame2==NULL) return -1 ;
01510 
01511     /* Get property lists */
01512     if ((plist1=cpl_propertylist_load(cpl_frame_get_filename(frame1),
01513                     0)) == NULL) {
01514         cpl_msg_error(__func__, "getting header from reference frame");
01515         return -1 ;
01516     }
01517     if ((plist2=cpl_propertylist_load(cpl_frame_get_filename(frame2),
01518                     0)) == NULL) {
01519         cpl_msg_error(__func__, "getting header from reference frame");
01520         cpl_propertylist_delete(plist1) ;
01521         return -1 ;
01522     }
01523 
01524     /* Test status */
01525     if (cpl_error_get_code()) {
01526         cpl_propertylist_delete(plist1) ;
01527         cpl_propertylist_delete(plist2) ;
01528         return -1 ;
01529     }
01530 
01531     comparison = 1 ;
01532 
01533     /* Compare filters */
01534     sval1 = hawki_pfits_get_filter(plist1) ;
01535     sval2 = hawki_pfits_get_filter(plist2) ;
01536     if (cpl_error_get_code()) {
01537         cpl_msg_error(__func__, "cannot get the filter");
01538         cpl_propertylist_delete(plist1) ;
01539         cpl_propertylist_delete(plist2) ;
01540         return -1 ;
01541     }
01542     if (strcmp(sval1, sval2)) comparison = 0 ;
01543 
01544     /* Compare DITs */
01545     dval1 = hawki_pfits_get_dit(plist1) ;
01546     dval2 = hawki_pfits_get_dit(plist2) ;
01547     if (cpl_error_get_code()) {
01548         cpl_msg_error(__func__, "cannot get the DIT");
01549         cpl_propertylist_delete(plist1) ;
01550         cpl_propertylist_delete(plist2) ;
01551         return -1 ;
01552     }
01553     if (fabs(dval1-dval2) > 1e-4) comparison = 0 ;
01554 
01555     cpl_propertylist_delete(plist1) ;
01556     cpl_propertylist_delete(plist2) ;
01557     return comparison ;
01558 }
01559 
01560 /*----------------------------------------------------------------------------*/
01567 /*----------------------------------------------------------------------------*/
01568 static cpl_imagelist * hawki_cal_flat_merge_bpms
01569 (const cpl_frame *   bpm_orig,
01570  cpl_imagelist   *   bpm_to_merge)
01571 {
01572     cpl_imagelist   *   merged ;
01573     cpl_imagelist   *   bpm_orig_im;
01574     cpl_image       *   tmp ;
01575     int                 i ;
01576 
01577     /* Test entries */
01578     if (bpm_orig==NULL || bpm_to_merge ==NULL) return NULL ;
01579 
01580     /* Create merged */
01581     merged = cpl_imagelist_new() ;
01582 
01583     /* Load the bpm_file */
01584     bpm_orig_im = hawki_load_frame(bpm_orig, CPL_TYPE_INT);
01585     if(bpm_orig_im == NULL)
01586     {
01587         cpl_msg_error(__func__, "Cannot read existing bpm %s",
01588                       cpl_frame_get_filename(bpm_orig));
01589         cpl_imagelist_delete(merged);
01590         return NULL;
01591     }
01592 
01593     /* Loop on the chips */
01594     for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) {
01595 
01596         /* Merge */
01597         tmp = cpl_image_add_create(cpl_imagelist_get(bpm_orig_im, i),
01598                                    cpl_imagelist_get(bpm_to_merge, i));
01599         cpl_image_multiply(cpl_imagelist_get(bpm_orig_im, i),
01600                            cpl_imagelist_get(bpm_to_merge, i));
01601         cpl_image_subtract(tmp, cpl_imagelist_get(bpm_orig_im, i)) ;
01602 
01603         /* Store */
01604         cpl_imagelist_set(merged, tmp, i) ;
01605     }
01606 
01607     /* Clean-up and return */
01608     cpl_imagelist_delete(bpm_orig_im);
01609     return merged ;
01610 }
01611 
01612 /*----------------------------------------------------------------------------*/
01623 /*----------------------------------------------------------------------------*/
01624 static int hawki_cal_flat_select
01625 (cpl_vector  *   meds,
01626  cpl_vector  *   rms,
01627  int             auto_flag,
01628  int             auto_max_bins,
01629  double          min_level,
01630  double          max_level,
01631  double          max_rms,
01632  int             min_nframes,
01633  cpl_vector  *   selection)
01634 {
01635     double      *   pselection ;
01636     double      *   pmeds ;
01637     double      *   prms ;
01638     int             nvals ;
01639     int             nsel ;
01640     double          first_val, last_val, bin_val, dist, dist_min;
01641     double          bin_low_val;
01642     double          bin_up_val;
01643     int             nbins, ind_closest ;
01644     int             ibin;
01645     int             ival;
01646     
01647     /* Test entries */
01648     if (meds == NULL) return -1;
01649     if (rms == NULL) return -1;
01650     if (selection == NULL) return -1;
01651     if(cpl_vector_get_size(meds) != cpl_vector_get_size(selection))
01652     {
01653         cpl_msg_error(__func__, 
01654                       "The meds and selection vectors have not the same size");
01655         return -1;
01656     }
01657 
01658     /* Initialise. All the frames are selected by default */
01659     nvals = cpl_vector_get_size(meds);
01660     pmeds = cpl_vector_get_data(meds);
01661     prms = cpl_vector_get_data(rms);
01662     cpl_vector_fill(selection, 1.0);  
01663 
01664     /* Pointer to selection */
01665     pselection = cpl_vector_get_data(selection) ;
01666 
01667     /* First select based on minimum level, if applies */
01668     if (min_level >= 0.0)
01669     {
01670         for (ival=0 ; ival<nvals ; ival++) 
01671         {
01672             if (pmeds[ival] < min_level)
01673                 pselection[ival] = -1.0 ;
01674         }
01675     }
01676 
01677     /* Second select based on maximum level, if applies */
01678     if (max_level >= 0.0)
01679     {
01680         for (ival=0 ; ival<nvals ; ival++) 
01681         {
01682             if (pmeds[ival] > max_level)
01683                 pselection[ival] = -1.0 ;
01684         }
01685     }
01686 
01687     /* Third select based on rms maximum level, if applies */
01688     if (max_rms >= 0.0)
01689     {
01690         for (ival=0 ; ival<nvals ; ival++) 
01691         {
01692             if (prms[ival] > max_rms)
01693                 pselection[ival] = -1.0 ;
01694         }
01695     }
01696 
01697     /* Apply the histogram selection */
01698     if (auto_flag)
01699     {
01700         /* Automatic  */
01701         cpl_vector * auto_selection;
01702         auto_selection = cpl_vector_new(nvals);
01703         cpl_vector_fill(auto_selection, -1.0);
01704         cpl_msg_info(__func__, "Automatic dynamic range selection");
01705         first_val = hawki_vector_get_min_select(meds, selection);
01706         last_val = hawki_vector_get_max_select(meds, selection);
01707 
01708         nbins = nvals ;
01709         if (auto_max_bins > 0 && auto_max_bins < nvals) nbins = auto_max_bins;
01710         for (ibin=0 ; ibin<nbins ; ibin++) 
01711         {
01712             if(ibin == 0)
01713                 bin_val = first_val + (last_val-first_val)*(ibin+1)/nbins ;
01714             else if(ibin == nbins - 1)
01715                 bin_val = first_val + (last_val-first_val)*(ibin)/nbins ;
01716             else
01717                 bin_val = first_val + (last_val-first_val)*(ibin+0.5)/nbins ;
01718             bin_low_val = first_val + (last_val-first_val)*(ibin)/nbins ;
01719             bin_up_val  = first_val + (last_val-first_val)*(ibin+1)/nbins ;
01720 
01721             /* Select the closest */
01722             dist_min = fabs(pmeds[0] - bin_val) ;
01723             ind_closest = -1;
01724             for (ival=0 ; ival<nvals ; ival++) 
01725             {
01726                 dist = fabs(pmeds[ival] - bin_val) ;
01727                 if (dist < dist_min && pselection[ival] != -1) 
01728                 {
01729                     dist_min = dist;
01730                     ind_closest = ival;
01731                 }
01732             }
01733             if(ind_closest != -1)
01734                 cpl_vector_set(auto_selection, ind_closest, 1.0);
01735         }
01736         /* Use the auto_selection */
01737         cpl_vector_copy(selection, auto_selection);
01738         cpl_vector_delete(auto_selection);
01739     }
01740     
01741     /* Print and count the selected frames */
01742     nsel = 0;
01743     cpl_msg_indent_more();
01744     for (ival=0 ; ival<nvals ; ival++) 
01745     {
01746         if(pselection[ival] != -1)
01747         {
01748             cpl_msg_info(__func__, "Selected frame %d", ival+1) ;
01749             nsel++;
01750         }
01751     }
01752     cpl_msg_indent_less();
01753     
01754     /* Check the number of selected against min_nframes */
01755     if (nsel == 0) {
01756         cpl_msg_error(__func__, "No frame selected. Check selection criteria");
01757         return -1;
01758     }
01759     if (min_nframes >= 0) {
01760         if (nsel < min_nframes) {
01761             cpl_msg_error(__func__, "Not enough selected frames (%d < %d)",
01762                 nsel, min_nframes) ;
01763             return -1;
01764         }
01765     }
01766     
01767     return 0;
01768 }
01769  
01770 /*----------------------------------------------------------------------------*/
01777 /*----------------------------------------------------------------------------*/
01778 static cpl_vector * hawki_cal_flat_extract_vector(
01779         cpl_vector  *   in,
01780         cpl_vector  *   selection)
01781 {
01782     int             nvals ;
01783     cpl_vector  *   selected ;
01784     double      *   pin ;
01785     double      *   pselection ;
01786     double      *   pselected ;
01787     int             nselected ;
01788     int             i ;
01789     
01790     /* Test entries */
01791     if (in == NULL) return NULL ;
01792     if (selection == NULL) return NULL ;
01793 
01794     /* Initialise */
01795     nvals = cpl_vector_get_size(in) ;
01796     nselected = 0 ;
01797     pin = cpl_vector_get_data(in) ;
01798     pselection = cpl_vector_get_data(selection) ;
01799 
01800     /* Test entries */
01801     if (nvals != cpl_vector_get_size(selection)) return NULL ; 
01802 
01803     /* Count the selected values */
01804     for (i=0 ; i<nvals ; i++) {
01805         if (pselection[i] > 0.0) nselected++ ;
01806     } 
01807     if (nselected == 0) return NULL ;
01808 
01809     /* Create the selected vector */
01810     selected = cpl_vector_new(nselected) ;
01811     pselected = cpl_vector_get_data(selected) ;
01812 
01813     nselected = 0 ;
01814     for (i=0 ; i<nvals ; i++) {
01815         if (pselection[i] > 0.0) {
01816             pselected[nselected] = pin[i] ;
01817             nselected++ ;
01818         }
01819     } 
01820     return selected ;
01821 }
01822 
01823 static int hawki_cal_flat_retrieve_input_param
01824 (cpl_parameterlist  *  parlist)
01825 {
01826     const char      * sval;
01827     cpl_parameter   * par;
01828     int               idet;
01829     
01830     /* Initialise */
01831     par = NULL ;
01832     for (idet=0 ; idet<HAWKI_NB_DETECTORS ; idet++) {
01833         hawki_cal_flat_outputs.nb_badpix[idet] = -1 ;
01834         hawki_cal_flat_outputs.med_stdev[idet] = -1.0 ;
01835         hawki_cal_flat_outputs.med_avg[idet] = -1.0 ;
01836         hawki_cal_flat_outputs.med_med[idet] = -1.0 ;
01837         hawki_cal_flat_outputs.med_min[idet] = -1.0 ;
01838         hawki_cal_flat_outputs.med_max[idet] = -1.0 ;
01839     }
01840 
01841     /* Retrieve input parameters */
01842     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.zone") ;
01843     sval = cpl_parameter_get_string(par) ;
01844     if (sscanf(sval, "%d,%d,%d,%d",
01845                     &hawki_cal_flat_config.llx,
01846                     &hawki_cal_flat_config.lly,
01847                     &hawki_cal_flat_config.urx,
01848                     &hawki_cal_flat_config.ury)!=4) 
01849     {
01850         return -1 ;
01851     }
01852     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.sigma_badres") ;
01853     hawki_cal_flat_config.sigma_badres = cpl_parameter_get_double(par) ;
01854     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.sigma_bpm") ;
01855     hawki_cal_flat_config.sigma_bpm = cpl_parameter_get_double(par) ;
01856     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.lowval_bpm") ;
01857     hawki_cal_flat_config.lowval_bpm = cpl_parameter_get_double(par) ;
01858     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.highval_bpm") ;
01859     hawki_cal_flat_config.highval_bpm = cpl_parameter_get_double(par) ;
01860     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.normalise") ;
01861     hawki_cal_flat_config.normalise = cpl_parameter_get_bool(par) ;
01862     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.second_pass") ;
01863     hawki_cal_flat_config.second_pass = cpl_parameter_get_bool(par) ;
01864     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.select_auto") ;
01865     hawki_cal_flat_config.select_auto = cpl_parameter_get_bool(par) ;
01866     par = cpl_parameterlist_find(parlist, 
01867             "hawki.hawki_cal_flat.select_auto_max_bins") ;
01868     hawki_cal_flat_config.select_auto_max_bins = cpl_parameter_get_int(par);
01869     par = cpl_parameterlist_find(parlist, 
01870             "hawki.hawki_cal_flat.select_min_level") ;
01871     sval = cpl_parameter_get_string(par);
01872     if (sscanf(sval, "%lf,%lf,%lf,%lf",
01873                &hawki_cal_flat_config.select_min_level[0],
01874                &hawki_cal_flat_config.select_min_level[1],
01875                &hawki_cal_flat_config.select_min_level[2],
01876                &hawki_cal_flat_config.select_min_level[3])!=4)
01877     {
01878         if (sscanf(sval, "%f", &hawki_cal_flat_config.select_min_level[0])!=1)
01879         {
01880             return -1;
01881         }
01882         else
01883         {
01884             hawki_cal_flat_config.select_min_level[1] = 
01885                 hawki_cal_flat_config.select_min_level[0];
01886             hawki_cal_flat_config.select_min_level[2] = 
01887                 hawki_cal_flat_config.select_min_level[0];
01888             hawki_cal_flat_config.select_min_level[3] = 
01889                 hawki_cal_flat_config.select_min_level[0];
01890         }
01891     }
01892     par = cpl_parameterlist_find(parlist, 
01893             "hawki.hawki_cal_flat.select_max_level") ;
01894     sval = cpl_parameter_get_string(par);
01895     if (sscanf(sval, "%lf,%lf,%lf,%lf",
01896                &hawki_cal_flat_config.select_max_level[0],
01897                &hawki_cal_flat_config.select_max_level[1],
01898                &hawki_cal_flat_config.select_max_level[2],
01899                &hawki_cal_flat_config.select_max_level[3])!=4)
01900     {
01901         if (sscanf(sval, "%f", &hawki_cal_flat_config.select_max_level[0])!=1)
01902         {
01903             return -1;
01904         }
01905         else
01906         {
01907             hawki_cal_flat_config.select_max_level[1] = 
01908                 hawki_cal_flat_config.select_max_level[0];
01909             hawki_cal_flat_config.select_max_level[2] = 
01910                 hawki_cal_flat_config.select_max_level[0];
01911             hawki_cal_flat_config.select_max_level[3] = 
01912                 hawki_cal_flat_config.select_max_level[0];
01913         }
01914     }
01915     par = cpl_parameterlist_find(parlist, 
01916             "hawki.hawki_cal_flat.select_max_rms") ;
01917     sval = cpl_parameter_get_string(par);
01918     if (sscanf(sval, "%lf,%lf,%lf,%lf",
01919                hawki_cal_flat_config.select_max_rms,
01920                hawki_cal_flat_config.select_max_rms+1,
01921                hawki_cal_flat_config.select_max_rms+2,
01922                hawki_cal_flat_config.select_max_rms+3)!=4)
01923     {
01924         if (sscanf(sval, "%f", &hawki_cal_flat_config.select_max_rms[0])!=1)
01925         {
01926             return -1;
01927         }
01928         else
01929         {
01930             hawki_cal_flat_config.select_max_rms[1] = 
01931                 hawki_cal_flat_config.select_max_rms[0];
01932             hawki_cal_flat_config.select_max_rms[2] = 
01933                 hawki_cal_flat_config.select_max_rms[0];
01934             hawki_cal_flat_config.select_max_rms[3] = 
01935                 hawki_cal_flat_config.select_max_rms[0];
01936         }
01937     }
01938     par = cpl_parameterlist_find(parlist, 
01939             "hawki.hawki_cal_flat.select_min_nframes") ;
01940     hawki_cal_flat_config.select_min_nframes = cpl_parameter_get_int(par) ;
01941     par = cpl_parameterlist_find(parlist, "hawki.hawki_cal_flat.extra_stats") ;
01942     hawki_cal_flat_config.extra_stats = cpl_parameter_get_bool(par) ;
01943 
01944     if(hawki_cal_flat_config.select_auto_max_bins != -1 &&
01945             !hawki_cal_flat_config.select_auto)
01946     {
01947             cpl_msg_error(__func__,"Max bins is only allowed with select_auto");
01948             return -1;
01949     }
01950         
01951     return 0;
01952 }