HAWKI Pipeline Reference Manual 1.8.12
hawki_step_basic_calib.c
00001 /* $Id: hawki_step_basic_calib.c,v 1.13 2011/01/31 11:03:23 cgarcia Exp $
00002  *
00003  * This file is part of the HAWKI Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: cgarcia $
00023  * $Date: 2011/01/31 11:03:23 $
00024  * $Revision: 1.13 $
00025  * $Name: hawki-1_8_12 $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033                                 Includes
00034  -----------------------------------------------------------------------------*/
00035 
00036 #include <string.h>
00037 #include <math.h>
00038 #include <cpl.h>
00039 
00040 #include "irplib_utils.h"
00041 #include "hawki_utils.h"
00042 #include "hawki_calib.h"
00043 #include "hawki_load.h"
00044 #include "hawki_save.h"
00045 #include "hawki_pfits.h"
00046 #include "hawki_dfs.h"
00047 
00048 /*-----------------------------------------------------------------------------
00049                                 Define
00050  -----------------------------------------------------------------------------*/
00051 
00052 #define NEGLIG_OFF_DIFF     0.1
00053 #define SQR(x) ((x)*(x))
00054 
00055 /*-----------------------------------------------------------------------------
00056                             Functions prototypes
00057  -----------------------------------------------------------------------------*/
00058 
00059 static int hawki_step_basic_calib_create(cpl_plugin *) ;
00060 static int hawki_step_basic_calib_exec(cpl_plugin *) ;
00061 static int hawki_step_basic_calib_destroy(cpl_plugin *) ;
00062 static int hawki_step_basic_calib(cpl_parameterlist *, cpl_frameset *) ;
00063 
00064 static int hawki_step_basic_calib_applycal_qc_save
00065 (cpl_frameset      *  raw_target,
00066  const cpl_frame   *  flat,
00067  const cpl_frame   *  dark,
00068  const cpl_frame   *  bpm,
00069  const char        *  filename_postfix,
00070  const char        *  procat,
00071  const char        *  protype,
00072  cpl_parameterlist *  recipe_parlist,
00073  cpl_frameset      *  recipe_framelist);
00074 static void hawki_step_basic_calib_qc(void);
00075 static int hawki_step_basic_calib_save
00076 (cpl_imagelist     *  reduced,
00077  const char        *  filename_postfix,
00078  const char        *  procat,
00079  const char        *  protype,
00080  int                  iserie,
00081  cpl_frameset      *  used_frameset,
00082  cpl_parameterlist *  recipe_parlist,
00083  cpl_frameset      *  recipe_framelist);
00084 
00085 /*-----------------------------------------------------------------------------
00086                             Static variables
00087  -----------------------------------------------------------------------------*/
00088 
00089 static char hawki_step_basic_calib_description[] =
00090 "hawki_step_basic_calib -- hawki basic reduction utility (flat, dark).\n"
00091 "The files listed in the Set Of Frames (sof-file) must be tagged:\n"
00092 "raw-file.fits "HAWKI_IMG_JITTER_RAW" or\n"
00093 "raw-file.fits "HAWKI_IMG_JITTER_SKY_RAW" \n"
00094 "flat-file.fits "HAWKI_CALPRO_FLAT" \n"
00095 "dark-file.fits "HAWKI_CALPRO_DARK" \n"
00096 "bpm-file.fits "HAWKI_CALPRO_BPM" \n";
00097 
00098 /*-----------------------------------------------------------------------------
00099                                 Functions code
00100  -----------------------------------------------------------------------------*/
00101 
00102 /*----------------------------------------------------------------------------*/
00110 /*----------------------------------------------------------------------------*/
00111 int cpl_plugin_get_info(cpl_pluginlist * list)
00112 {
00113     cpl_recipe  *   recipe = cpl_calloc(1, sizeof(*recipe)) ;
00114     cpl_plugin  *   plugin = &recipe->interface ;
00115 
00116     cpl_plugin_init(plugin,
00117                     CPL_PLUGIN_API,
00118                     HAWKI_BINARY_VERSION,
00119                     CPL_PLUGIN_TYPE_RECIPE,
00120                     "hawki_step_basic_calib",
00121                     "Basic reduction recipe",
00122                     hawki_step_basic_calib_description,
00123                     "Cesar Enrique Garcia Dabo",
00124                     "cgarcia@eso.org",
00125                     hawki_get_license(),
00126                     hawki_step_basic_calib_create,
00127                     hawki_step_basic_calib_exec,
00128                     hawki_step_basic_calib_destroy) ;
00129 
00130     cpl_pluginlist_append(list, plugin) ;
00131     
00132     return 0;
00133 }
00134 
00135 /*----------------------------------------------------------------------------*/
00144 /*----------------------------------------------------------------------------*/
00145 static int hawki_step_basic_calib_create(cpl_plugin * plugin)
00146 {
00147     cpl_recipe      * recipe ;
00148     /* cpl_parameter   * p ; */
00149 
00150     /* Get the recipe out of the plugin */
00151     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00152         recipe = (cpl_recipe *)plugin ;
00153     else return -1 ;
00154 
00155     /* Create the parameters list in the cpl_recipe object */
00156     recipe->parameters = cpl_parameterlist_new() ;
00157 
00158     /* Fill the parameters list */
00159     /* None.. */
00160 
00161     /* Return */
00162     return 0;
00163 }
00164 
00165 /*----------------------------------------------------------------------------*/
00171 /*----------------------------------------------------------------------------*/
00172 static int hawki_step_basic_calib_exec(cpl_plugin * plugin)
00173 {
00174     cpl_recipe  *   recipe ;
00175 
00176     /* Get the recipe out of the plugin */
00177     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00178         recipe = (cpl_recipe *)plugin ;
00179     else return -1 ;
00180 
00181     /* Issue a banner */
00182     hawki_print_banner();
00183 
00184     return hawki_step_basic_calib(recipe->parameters, recipe->frames) ;
00185 }
00186 
00187 /*----------------------------------------------------------------------------*/
00193 /*----------------------------------------------------------------------------*/
00194 static int hawki_step_basic_calib_destroy(cpl_plugin * plugin)
00195 {
00196     cpl_recipe  *   recipe ;
00197 
00198     /* Get the recipe out of the plugin */
00199     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00200         recipe = (cpl_recipe *)plugin ;
00201     else return -1 ;
00202 
00203     cpl_parameterlist_delete(recipe->parameters) ;
00204     return 0 ;
00205 }
00206 
00207 /*----------------------------------------------------------------------------*/
00214 /*----------------------------------------------------------------------------*/
00215 static int hawki_step_basic_calib(
00216         cpl_parameterlist   *   parlist, 
00217         cpl_frameset        *   framelist)
00218 {
00219     const cpl_frame  * flat;
00220     const cpl_frame  * dark;
00221     const cpl_frame  * bpm;
00222     cpl_frameset     * objframes ;
00223     cpl_frameset     * skyframes ;
00224 
00225     /* Identify the RAW and CALIB frames in the input frameset */
00226     if (hawki_dfs_set_groups(framelist)) {
00227         cpl_msg_error(__func__, "Cannot identify RAW and CALIB frames") ;
00228         return -1 ;
00229     }
00230 
00231     /* Retrieve calibration data */
00232     cpl_msg_info(__func__, "Identifying calibration data");
00233     flat   = cpl_frameset_find_const(framelist, HAWKI_CALPRO_FLAT);
00234     dark   = cpl_frameset_find_const(framelist, HAWKI_CALPRO_DARK);
00235     bpm    = cpl_frameset_find_const(framelist, HAWKI_CALPRO_BPM);
00236     if (flat == NULL && dark == NULL && bpm == NULL)
00237     {
00238         cpl_msg_error(__func__, "No calibration data provided (%s and/or %s and/or %s)",
00239                 HAWKI_CALPRO_FLAT, HAWKI_CALPRO_DARK, HAWKI_CALPRO_BPM);
00240         return -1 ;
00241     }
00242     
00243     /* Retrieve raw frames */
00244     cpl_msg_info(__func__, "Identifying objects and sky data");
00245     objframes = hawki_extract_frameset(framelist, HAWKI_IMG_JITTER_RAW) ;
00246     /* Retrieve sky frames */
00247     skyframes = hawki_extract_frameset(framelist, HAWKI_IMG_JITTER_SKY_RAW) ;
00248     if (objframes == NULL && skyframes == NULL)
00249     {
00250         cpl_msg_error(__func__, "Cannot find objs (%s) or sky frames (%s) in the input list",
00251                 HAWKI_IMG_JITTER_RAW, HAWKI_IMG_JITTER_SKY_RAW);
00252         return -1 ;
00253     }
00254     
00255     /* Applying the calibrations, QC and saving the results */
00256     if (objframes != NULL)
00257     {
00258         const char * procat  = HAWKI_CALPRO_BASICCALIBRATED;
00259         const char * protype = HAWKI_PROTYPE_BASICCALIBRATED;
00260         cpl_msg_info(__func__, "Apply the basic reduction to object frames");
00261         hawki_step_basic_calib_applycal_qc_save
00262             (objframes, flat, dark, bpm, "obj", procat, protype,
00263              parlist, framelist);
00264         cpl_frameset_delete(objframes);
00265     }
00266     if (skyframes != NULL)
00267     {
00268         const char * procat  = HAWKI_CALPRO_SKY_BASICCALIBRATED;
00269         const char * protype = HAWKI_PROTYPE_SKY_BASICCALIBRATED;
00270         cpl_msg_info(__func__, "Apply the basic reduction to sky frames");
00271         hawki_step_basic_calib_applycal_qc_save
00272             (skyframes, flat, dark, bpm, "sky", procat, protype,
00273              parlist, framelist);
00274         cpl_frameset_delete(skyframes);
00275     }
00276 
00277     /* return */
00278     if (cpl_error_get_code()) return -1 ;
00279     else return 0 ;
00280 }
00281 
00282 /*----------------------------------------------------------------------------*/
00292 /*----------------------------------------------------------------------------*/
00293 static int hawki_step_basic_calib_applycal_qc_save
00294 (cpl_frameset      *  raw_target,
00295  const cpl_frame   *  flat,
00296  const cpl_frame   *  dark,
00297  const cpl_frame   *  bpm,
00298  const char        *  filename_postfix,
00299  const char        *  procat,
00300  const char        *  protype,
00301  cpl_parameterlist *  recipe_parlist,
00302  cpl_frameset      *  recipe_framelist)
00303 {
00304     cpl_imagelist    *  flat_images;
00305     cpl_imagelist    *  dark_images;
00306     cpl_imagelist    *  bpm_images;
00307     cpl_frameset     *  calib_frameset;  
00308     cpl_propertylist *  plist ;
00309     cpl_errorstate      error_prevstate = cpl_errorstate_get();
00310     
00311     double              science_dit;
00312     int                 iframe;
00313     int                 ntarget;
00314 
00315     /* Initializing the pointers */
00316     flat_images = NULL;
00317     dark_images = NULL;
00318     bpm_images  = NULL;
00319     
00320     /* Indentation */
00321     cpl_msg_indent_more();
00322     
00323     /* Initializating the calibration frameset */
00324     calib_frameset = cpl_frameset_new();
00325 
00326     /* Loading the calibration files */
00327     cpl_msg_info(__func__, "Loading the calibration data") ;
00328     if(flat != NULL)
00329     {
00330         flat_images = hawki_load_frame(flat, CPL_TYPE_FLOAT);
00331         if(flat_images == NULL)
00332         {
00333             cpl_msg_error(__func__, "Error reading flat") ;
00334             cpl_frameset_delete(calib_frameset);
00335             return -1;
00336         }
00337         cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(flat));
00338     }
00339     if(dark != NULL)
00340     {
00341         dark_images = hawki_load_frame(dark, CPL_TYPE_FLOAT);
00342         if(dark_images == NULL)
00343         {
00344             cpl_msg_error(__func__, "Error reading dark") ;
00345             cpl_imagelist_delete(flat_images);
00346             cpl_frameset_delete(calib_frameset);
00347             return -1;
00348         }
00349         cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(dark));
00350     }
00351     if(bpm != NULL)
00352     {
00353         bpm_images = hawki_load_frame(bpm, CPL_TYPE_INT);
00354         if(bpm_images == NULL)
00355         {
00356             cpl_msg_error(__func__, "Error reading bpm") ;
00357             cpl_imagelist_delete(flat_images);
00358             cpl_imagelist_delete(dark_images);
00359             cpl_frameset_delete(calib_frameset);
00360             return -1;
00361         }
00362         cpl_frameset_insert(calib_frameset, cpl_frame_duplicate(bpm));
00363     }
00364 
00365     /* Multiply the dark image by the science exposure time */
00366     if(dark != NULL)
00367     {
00368         if ((plist=cpl_propertylist_load
00369                 (cpl_frame_get_filename
00370                  (cpl_frameset_get_first_const(raw_target)), 0)) == NULL) 
00371         {
00372             cpl_msg_error(__func__, "Cannot get header from frame");
00373             cpl_imagelist_delete(flat_images);
00374             cpl_imagelist_delete(dark_images);
00375             cpl_frameset_delete(calib_frameset);
00376             return -1 ;
00377         }
00378         science_dit = hawki_pfits_get_dit(plist);
00379         cpl_imagelist_multiply_scalar(dark_images, science_dit);
00380         cpl_propertylist_delete(plist);
00381     }
00382 
00383     /* Loop on the number of frames */
00384     ntarget = cpl_frameset_get_size(raw_target);
00385     cpl_msg_info(__func__, "Looping the science frames: %d frames", ntarget);
00386     for( iframe = 0 ; iframe < ntarget ; ++iframe)
00387     {
00388         /* Local storage variables */
00389         cpl_frame     * target_frame;
00390         cpl_imagelist * target_images;
00391         cpl_frameset  * used_frameset;
00392         target_images = NULL;
00393 
00394         /* Loading the target */
00395         cpl_msg_indent_more();
00396         cpl_msg_info(__func__, "Loading frame %d", iframe+1) ;
00397         target_frame = cpl_frameset_get_frame(raw_target, iframe);
00398         if(target_frame != NULL)
00399             target_images = hawki_load_frame(target_frame, CPL_TYPE_FLOAT);
00400         if(target_images == NULL)
00401         {
00402             cpl_msg_error(__func__, "Error reading frame") ;
00403             cpl_imagelist_delete(flat_images);
00404             cpl_imagelist_delete(dark_images);
00405             cpl_imagelist_delete(bpm_images);
00406             return -1;
00407         }
00408         /* Creating the used frameset */
00409         used_frameset = cpl_frameset_duplicate(calib_frameset);
00410         cpl_frameset_insert(used_frameset, cpl_frame_duplicate(target_frame));
00411     
00412         /* TODO: Creating the variance array */
00413         /* cpl_create_variance_image(); */
00414         
00415         /* Applying the calibrations */
00416         cpl_msg_info(__func__, "Calibrating frame") ;
00417         if (hawki_flat_dark_bpm_imglist_calib
00418                 (target_images, flat_images, dark_images, bpm_images) == -1) 
00419         {
00420             cpl_msg_error(__func__, "Cannot calibrate frame") ;
00421             cpl_imagelist_delete(flat_images);
00422             cpl_imagelist_delete(dark_images);
00423             cpl_imagelist_delete(bpm_images);
00424             cpl_imagelist_delete(target_images);
00425             cpl_frameset_delete(used_frameset);            
00426             cpl_frameset_delete(calib_frameset);
00427             cpl_msg_indent_less() ;
00428             cpl_msg_indent_less() ;
00429             return -1 ;
00430         }
00431 
00432         /* Compute quality control */
00433         hawki_step_basic_calib_qc();
00434         
00435         /* Save the products */
00436         cpl_msg_info(__func__, "Save the products") ;
00437         if (hawki_step_basic_calib_save
00438                 (target_images, filename_postfix, procat, protype, iframe,
00439                  used_frameset, recipe_parlist, 
00440                  recipe_framelist) == -1)
00441         {
00442             cpl_msg_error(__func__, "Cannot save the products") ;
00443             cpl_imagelist_delete(flat_images);
00444             cpl_imagelist_delete(dark_images);
00445             cpl_imagelist_delete(bpm_images);
00446             cpl_imagelist_delete(target_images);
00447             cpl_frameset_delete(used_frameset);
00448             cpl_frameset_delete(calib_frameset);
00449             cpl_msg_indent_less() ;
00450             cpl_msg_indent_less() ;
00451             return -1 ;
00452         }
00453 
00454         /* Delete the target and the used frameset */
00455         cpl_imagelist_delete(target_images);
00456         cpl_frameset_delete(used_frameset);
00457         cpl_msg_indent_less();
00458     }
00459     
00460     cpl_msg_indent_less();
00461     /* Delete the calibration files */
00462     cpl_imagelist_delete(flat_images);
00463     cpl_imagelist_delete(dark_images);
00464     cpl_imagelist_delete(bpm_images);
00465     cpl_frameset_delete(calib_frameset);
00466     
00467     if(!cpl_errorstate_is_equal(error_prevstate))
00468     {
00469         cpl_msg_warning(__func__,"Probably some data could not be saved. "
00470                                  "Check permisions or disk space");
00471         cpl_errorstate_set(CPL_ERROR_NONE);
00472         return 1;
00473     }
00474     return 0;
00475 }
00476 
00477 /*----------------------------------------------------------------------------*/
00484 /*----------------------------------------------------------------------------*/
00485 static void hawki_step_basic_calib_qc(void)
00486 {
00487     /* Compute QC parameters from the combined image */
00488     //cpl_msg_info(__func__, "Compute QC parameters from the reduced images") ;
00489 }
00490         
00491 /*----------------------------------------------------------------------------*/
00501 /*----------------------------------------------------------------------------*/
00502 static int hawki_step_basic_calib_save
00503 (cpl_imagelist     *  reduced,
00504  const char        *  filename_postfix,
00505  const char        *  procat,
00506  const char        *  protype,
00507  int                  iserie,
00508  cpl_frameset      *  used_frameset,
00509  cpl_parameterlist *  recipe_parlist,
00510  cpl_frameset      *  recipe_framelist)
00511 {
00512     const cpl_frame     *   raw_reference;
00513     cpl_propertylist    *   proplist;
00514     cpl_propertylist    **  extproplists;
00515     char                    filename[256];
00516     cpl_propertylist    *   inputlist ;
00517     int                     ext_nb ;
00518     const char          *   recipe_name = "hawki_step_basic_calib";
00519     int i;
00520 
00521     /* Get the reference frame (the raw frame) */
00522     raw_reference = irplib_frameset_get_first_from_group
00523         (used_frameset, CPL_FRAME_GROUP_RAW);
00524     
00525     /* Create the prop lists */
00526     cpl_msg_indent_more();
00527     proplist  = cpl_propertylist_new();
00528     inputlist = cpl_propertylist_load_regexp(
00529                     cpl_frame_get_filename(raw_reference), 0,
00530                                            HAWKI_HEADER_EXT_FORWARD, 0);
00531     cpl_propertylist_append(proplist, inputlist);
00532     cpl_propertylist_delete(inputlist);
00533     extproplists = cpl_malloc(HAWKI_NB_DETECTORS * sizeof(cpl_propertylist*));
00534     for (i=0 ; i<HAWKI_NB_DETECTORS ; i++)
00535     {
00536         /* Get the extension number */
00537         ext_nb=hawki_get_ext_from_detector
00538             (cpl_frame_get_filename(raw_reference), i+1);
00539 
00540         /* Allocate this pro perty list */
00541         extproplists[i] = cpl_propertylist_new();
00542 
00543         /* Propagate some keywords from input raw frame extensions */
00544         inputlist = cpl_propertylist_load_regexp(
00545                 cpl_frame_get_filename(raw_reference), ext_nb,
00546                 HAWKI_HEADER_EXT_FORWARD, 0);
00547         cpl_propertylist_append(extproplists[i], inputlist);
00548         cpl_propertylist_delete(inputlist);
00549         inputlist = cpl_propertylist_load_regexp(
00550                 cpl_frame_get_filename(raw_reference), ext_nb,
00551                 HAWKI_HEADER_WCS, 0);
00552         cpl_propertylist_append(extproplists[i], inputlist);
00553         cpl_propertylist_delete(inputlist);
00554     }
00555     
00556     /* Write the image */
00557     snprintf(filename, 256, "hawki_step_basic_calib_%s%03d.fits",
00558              filename_postfix, iserie+1);
00559     hawki_imagelist_save(recipe_framelist,
00560                          recipe_parlist,
00561                          used_frameset,
00562                          reduced,
00563                          recipe_name,
00564                          procat, 
00565                          protype, 
00566                          (const cpl_propertylist*)proplist,
00567                          (const cpl_propertylist**)extproplists,
00568                          filename);
00569 
00570     /* Free and return */
00571     cpl_msg_indent_less();
00572     cpl_propertylist_delete(proplist) ;
00573     for (i=0 ; i<HAWKI_NB_DETECTORS ; i++) 
00574     {
00575         cpl_propertylist_delete(extproplists[i]) ;
00576     }
00577     cpl_free(extproplists) ;
00578     return  0;
00579 }