vircam_standard_process.c

00001 /* $Id: vircam_standard_process.c,v 1.43 2012/01/16 12:32:18 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2012/01/16 12:32:18 $
00024  * $Revision: 1.43 $
00025  * $Name: vcam-1_3_2 $
00026  */
00027 
00028 /* Includes */
00029 
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033 
00034 #include <stdio.h>
00035 #include <math.h>
00036 #include <string.h>
00037 #include <cpl.h>
00038 
00039 #include "vircam_utils.h"
00040 #include "vircam_mask.h"
00041 #include "vircam_pfits.h"
00042 #include "vircam_dfs.h"
00043 #include "vircam_mods.h"
00044 #include "vircam_fits.h"
00045 #include "vircam_tfits.h"
00046 #include "vircam_jmp_utils.h"
00047 #include "vircam_paf.h"
00048 
00049 /* Function prototypes */
00050 
00051 static int vircam_standard_process_create(cpl_plugin *);
00052 static int vircam_standard_process_exec(cpl_plugin *);
00053 static int vircam_standard_process_destroy(cpl_plugin *);
00054 static int vircam_standard_process(cpl_parameterlist *,cpl_frameset *);
00055 static cpl_propertylist *vircam_standard_process_dummyqc(int type);
00056 
00057 
00058 static char vircam_standard_process_description[] =
00059 "vircam_standard_process -- VIRCAM standard field processing recipe.\n\n"
00060 "Process a complete pawprint for standard fields in VIRCAM data.\n"
00061 "Remove instrumental signature, interleave microstep sequences (if done),\n"
00062 "combine jitters, photometrically and astrometrically calibrate the pawprint\n"
00063 "image. Photometric calibration can be done with 2mass and 1 other source\n\n"
00064 "The program accepts the following files in the SOF:\n\n"
00065 "    Tag                   Description\n"
00066 "    -----------------------------------------------------------------------\n"
00067 "    %-21s A list of raw science images\n"
00068 "    %-21s A master dark frame\n"
00069 "    %-21s A master twilight flat frame\n"
00070 "    %-21s A channel table\n"
00071 "    %-21s A photometric calibration table\n"
00072 "    %-21s A readnoise/gain file\n"
00073 "    %-21s A master confidence map or\n"
00074 "    %-21s A master bad pixel mask\n"
00075 "    %-21s A master 2mass catalogue index\n"
00076 "    %-21s A second reference catalogue index\n"
00077 "All of the above are required\n"
00078 "\n";
00079 
00223 /* Function code */
00224 
00225 /*---------------------------------------------------------------------------*/
00233 /*---------------------------------------------------------------------------*/
00234 
00235 int cpl_plugin_get_info(cpl_pluginlist *list) {
00236     cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
00237     cpl_plugin  *plugin = &recipe->interface;
00238     char alldesc[SZ_ALLDESC];
00239     (void)snprintf(alldesc,SZ_ALLDESC,vircam_standard_process_description,
00240                    VIRCAM_STD_OBJECT_RAW,VIRCAM_CAL_DARK,
00241                    VIRCAM_CAL_TWILIGHT_FLAT,VIRCAM_CAL_CHANTAB,
00242                    VIRCAM_CAL_PHOTTAB,VIRCAM_CAL_READGAINFILE,VIRCAM_CAL_CONF,
00243                    VIRCAM_CAL_BPM,VIRCAM_CAL_2MASS,VIRCAM_CAL_REFCAT);
00244 
00245     cpl_plugin_init(plugin,
00246                     CPL_PLUGIN_API,
00247                     VIRCAM_BINARY_VERSION,
00248                     CPL_PLUGIN_TYPE_RECIPE,
00249                     "vircam_standard_process",
00250                     "VIRCAM standard field processing recipe",
00251                     alldesc,
00252                     "Jim Lewis",
00253                     "jrl@ast.cam.ac.uk",
00254                     vircam_get_license(),
00255                     vircam_standard_process_create,
00256                     vircam_standard_process_exec,
00257                     vircam_standard_process_destroy);
00258 
00259     cpl_pluginlist_append(list,plugin);
00260 
00261     return(0);
00262 }
00263 
00264 /*---------------------------------------------------------------------------*/
00273 /*---------------------------------------------------------------------------*/
00274 
00275 static int vircam_standard_process_create(cpl_plugin *plugin) {
00276     cpl_recipe      *recipe;
00277     cpl_parameter   *p;
00278 
00279     /* Get the recipe out of the plugin */
00280 
00281     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00282         recipe = (cpl_recipe *)plugin;
00283     else 
00284         return(-1);
00285 
00286     /* Create the parameters list in the cpl_recipe object */
00287 
00288     recipe->parameters = cpl_parameterlist_new();
00289 
00290     /* Fill in the minimum object size */
00291 
00292     p = cpl_parameter_new_value("vircam.vircam_standard_process.ipix",
00293                                 CPL_TYPE_INT,
00294                                 "Minimum pixel area for each detected object",
00295                                 "vircam.vircam_standard_process",5);
00296     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ipix");
00297     cpl_parameterlist_append(recipe->parameters,p);
00298 
00299     /* Fill in the detection threshold parameter */
00300 
00301     p = cpl_parameter_new_value("vircam.vircam_standard_process.thresh",
00302                                 CPL_TYPE_DOUBLE,
00303                                 "Detection threshold in sigma above sky",
00304                                 "vircam.vircam_standard_process",2.0);
00305     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"thr");
00306     cpl_parameterlist_append(recipe->parameters,p);
00307 
00308     /* Fill in flag to use deblending software or not */
00309 
00310     p = cpl_parameter_new_value("vircam.vircam_standard_process.icrowd",
00311                                 CPL_TYPE_BOOL,"Use deblending?",
00312                                 "vircam.vircam_standard_process",1);
00313     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"icrowd");
00314     cpl_parameterlist_append(recipe->parameters,p);
00315 
00316     /* Fill in core radius */
00317 
00318     p = cpl_parameter_new_value("vircam.vircam_standard_process.rcore",
00319                                 CPL_TYPE_DOUBLE,"Value of Rcore in pixels",
00320                                 "vircam.vircam_standard_process",3.0);
00321     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"rcore");
00322     cpl_parameterlist_append(recipe->parameters,p);
00323 
00324     /* Fill in background smoothing box size */
00325 
00326     p = cpl_parameter_new_value("vircam.vircam_standard_process.nbsize",
00327                                 CPL_TYPE_INT,"Background smoothing box size",
00328                                 "vircam.vircam_standard_process",64);
00329     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nb");
00330     cpl_parameterlist_append(recipe->parameters,p);
00331 
00332     /* Fill in flag to use save the output catalogue */
00333 
00334     p = cpl_parameter_new_value("vircam.vircam_standard_process.savecat",
00335                                 CPL_TYPE_BOOL,"Save catalogue?",
00336                                 "vircam.vircam_standard_process",1);
00337     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"savecat");
00338     cpl_parameterlist_append(recipe->parameters,p);
00339 
00340 
00341     /* Fill in flag to destripe the images */
00342 
00343     p = cpl_parameter_new_value("vircam.vircam_standard_process.destripe",
00344                                 CPL_TYPE_BOOL,"Destripe images?",
00345                                 "vircam.vircam_standard_process",1);
00346     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"destripe");
00347     cpl_parameterlist_append(recipe->parameters,p);
00348 
00349     /* Fill in flag to correct sky background */
00350 
00351     p = cpl_parameter_new_value("vircam.vircam_standard_process.skycor",
00352                                 CPL_TYPE_BOOL,"Do sky correction?",
00353                                 "vircam.vircam_standard_process",1);
00354     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"skycor");
00355     cpl_parameterlist_append(recipe->parameters,p);
00356 
00357     /* Fill in flag to save simple images */
00358 
00359     p = cpl_parameter_new_value("vircam.vircam_standard_process.savesimple",
00360                                 CPL_TYPE_BOOL,"Save simple images?",
00361                                 "vircam.vircam_standard_process",0);
00362     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"savesimple");
00363     cpl_parameterlist_append(recipe->parameters,p);
00364 
00365     /* Extension number of input frames to use */
00366 
00367     p = cpl_parameter_new_range("vircam.vircam_standard_process.extenum",
00368                                 CPL_TYPE_INT,
00369                                 "Extension number to be done, 0 == all",
00370                                 "vircam.vircam_standard_process",
00371                                 1,0,16);
00372     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
00373     cpl_parameterlist_append(recipe->parameters,p);
00374         
00375     /* Get out of here */
00376 
00377     return(0);
00378 }
00379     
00380     
00381 /*---------------------------------------------------------------------------*/
00387 /*---------------------------------------------------------------------------*/
00388 
00389 static int vircam_standard_process_exec(cpl_plugin *plugin) {
00390     cpl_recipe  *recipe;
00391 
00392     /* Get the recipe out of the plugin */
00393 
00394     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00395         recipe = (cpl_recipe *)plugin;
00396     else 
00397         return(-1);
00398 
00399     return(vircam_standard_process(recipe->parameters,recipe->frames));
00400 }
00401                                 
00402 /*---------------------------------------------------------------------------*/
00408 /*---------------------------------------------------------------------------*/
00409 
00410 static int vircam_standard_process_destroy(cpl_plugin *plugin) {
00411     cpl_recipe *recipe ;
00412 
00413     /* Get the recipe out of the plugin */
00414 
00415     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00416         recipe = (cpl_recipe *)plugin;
00417     else 
00418         return(-1);
00419 
00420     cpl_parameterlist_delete(recipe->parameters);
00421     return(0);
00422 }
00423 
00424 /*---------------------------------------------------------------------------*/
00431 /*---------------------------------------------------------------------------*/
00432 
00433 static int vircam_standard_process(cpl_parameterlist *parlist, 
00434                                    cpl_frameset *framelist) {
00435     const char *fctid="vircam_standard_process";
00436     cpl_parameter *p;
00437     int jst,jfn,status,j,i,retval,nusteps,isconf,live,ndit;
00438     cpl_size nlab;
00439     float readnoise,gain,gaincor_fac;
00440     vir_fits *ff;
00441     cpl_propertylist *ehu,*pp;
00442     cpl_frame *catindex;
00443 
00444     /* Check validity of input frameset */
00445 
00446     if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
00447         cpl_msg_error(fctid,"Input framelist NULL or has no input data");
00448         return(-1);
00449     }
00450 
00451     /* Check the files in the frameset */
00452 
00453     if (vircam_frameset_fexists(framelist) != VIR_OK) {
00454         cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
00455         return(-1);
00456     }
00457 
00458     /* Initialise some things */
00459 
00460     vircam_jmp_init();
00461     (void)strncpy(vircam_recipename,fctid,VIRCAM_PATHSZ);
00462     (void)snprintf(vircam_recipepaf,VIRCAM_PATHSZ,"VIRCAM/%s",fctid);
00463     recflag = RECSTD;
00464 
00465     /* Get the parameters */
00466 
00467     p = cpl_parameterlist_find(parlist,
00468                                "vircam.vircam_standard_process.ipix");
00469     vircam_jmp_config.ipix = cpl_parameter_get_int(p);
00470     p = cpl_parameterlist_find(parlist,
00471                                "vircam.vircam_standard_process.thresh");
00472     vircam_jmp_config.threshold = (float)cpl_parameter_get_double(p);
00473     p = cpl_parameterlist_find(parlist,
00474                                "vircam.vircam_standard_process.icrowd");
00475     vircam_jmp_config.icrowd = cpl_parameter_get_bool(p);
00476     p = cpl_parameterlist_find(parlist,
00477                                "vircam.vircam_standard_process.rcore");
00478     vircam_jmp_config.rcore = (float)cpl_parameter_get_double(p);
00479     p = cpl_parameterlist_find(parlist,
00480                                "vircam.vircam_standard_process.nbsize");
00481     vircam_jmp_config.nbsize = cpl_parameter_get_int(p);
00482     p = cpl_parameterlist_find(parlist,
00483                                "vircam.vircam_standard_process.savecat");
00484     vircam_jmp_config.savecat = cpl_parameter_get_bool(p);
00485     p = cpl_parameterlist_find(parlist,
00486                                "vircam.vircam_standard_process.destripe");
00487     vircam_jmp_config.destripe = cpl_parameter_get_bool(p);
00488     p = cpl_parameterlist_find(parlist,
00489                                "vircam.vircam_standard_process.skycor");
00490     vircam_jmp_config.skycor = cpl_parameter_get_bool(p);
00491     p = cpl_parameterlist_find(parlist,
00492                                "vircam.vircam_standard_process.savesimple");
00493     vircam_jmp_config.savesimple = cpl_parameter_get_bool(p);
00494     p = cpl_parameterlist_find(parlist,
00495                                "vircam.vircam_standard_process.extenum");
00496     vircam_jmp_config.extenum = cpl_parameter_get_int(p);
00497 
00498     /* Sort out raw from calib frames */
00499 
00500     if (vircam_dfs_set_groups(framelist) != VIR_OK) {
00501         cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
00502         vircam_jmp_tidy(0);
00503         return(-1);
00504     }
00505 
00506     /* Label the input frames */
00507 
00508     if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
00509                                            &nlab)) == NULL) {
00510         cpl_msg_error(fctid,"Cannot labelise the input frames");
00511         vircam_jmp_tidy(0);
00512         return(-1);
00513     }
00514 
00515     /* Get the input science frames */
00516 
00517     if ((ps.science_frames = 
00518          vircam_frameset_subgroup(framelist,ps.labels,nlab,
00519                                   VIRCAM_STD_OBJECT_RAW)) == NULL) {
00520         cpl_msg_error(fctid,"No science images to process!");
00521         vircam_jmp_tidy(0);
00522         return(-1);
00523     }
00524 
00525     /* Check to see if there is a master dark frame */
00526 
00527     if ((ps.master_dark = 
00528          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00529                                     VIRCAM_CAL_DARK)) == NULL) {
00530         cpl_msg_error(fctid,"No master dark found");
00531         vircam_jmp_tidy(0);
00532         return(-1);
00533     }
00534         
00535     /* Check to see if there is a master twilight flat frame */
00536 
00537     if ((ps.master_twilight_flat = 
00538          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00539                                     VIRCAM_CAL_TWILIGHT_FLAT)) == NULL) {
00540         cpl_msg_error(fctid,"No master twilight flat found");
00541         vircam_jmp_tidy(0);
00542         return(-1);
00543     }
00544         
00545     /* Get the gain corrections */
00546 
00547     status = VIR_OK;
00548     if (vircam_gaincor_calc(ps.master_twilight_flat,&i,&(ps.gaincors),
00549                             &status) != VIR_OK) {
00550         cpl_msg_error(fctid,"Error calculating gain corrections");
00551         vircam_jmp_tidy(0);
00552         return(-1);
00553     }
00554         
00555     /* Check to see if there is a readgain file */
00556 
00557     if ((ps.readgain_file = 
00558          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00559                                     VIRCAM_CAL_READGAINFILE)) == NULL) {
00560         cpl_msg_error(fctid,"No master readnoise/gain file found");
00561         vircam_jmp_tidy(0);
00562         return(-1);
00563     }
00564         
00565     /* Check to see if there is a master confidence map. If there isn't
00566        then look for a bad pixel mask that can be converted into a 
00567        confidence map (in an emergency) */
00568 
00569     isconf = 1;
00570     if ((ps.master_conf = 
00571          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00572                                     VIRCAM_CAL_CONF)) == NULL) {
00573         isconf = 0;
00574         if ((ps.master_conf = 
00575              vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00576                                         VIRCAM_CAL_BPM)) == NULL) {
00577             cpl_msg_error(fctid,"No master confidence map found");
00578             vircam_jmp_tidy(0);
00579             return(-1);
00580         }
00581     }
00582     ps.mask = vircam_mask_define(framelist,ps.labels,nlab);
00583         
00584     /* Check to see if there is a channel table */
00585 
00586     if ((ps.chantab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00587                                                  VIRCAM_CAL_CHANTAB)) == NULL) {
00588         cpl_msg_error(fctid,"No channel table found");
00589         vircam_jmp_tidy(0);
00590         return(-1);
00591     }
00592 
00593     /* Check to see if there is a photometric table */
00594 
00595     if ((ps.phottab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00596                                                  VIRCAM_CAL_PHOTTAB)) == NULL) {
00597         cpl_msg_error(fctid,"No photometric table found");
00598         vircam_jmp_tidy(0);
00599         return(-1);
00600     }
00601     if ((ps.tphottab = cpl_table_load(cpl_frame_get_filename(ps.phottab),1,0)) == NULL) {
00602         cpl_msg_error(fctid,"Unable to load photometric table");
00603         vircam_jmp_tidy(0);
00604         return(-1);
00605     }
00606 
00607     /* Is the 2mass index file specified? */
00608 
00609     if ((catindex = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00610                                                VIRCAM_CAL_2MASS)) == NULL) {
00611         cpl_msg_info(fctid,"No 2MASS index found -- cannot continue");
00612         vircam_jmp_tidy(0);
00613         return(-1);
00614     }
00615     
00616     /* Get catalogue parameters */
00617 
00618     if (vircam_catpars(catindex,&(ps.catpath),&(ps.catname)) == VIR_FATAL) {
00619         vircam_jmp_tidy(0);
00620         cpl_frame_delete(catindex);
00621         return(-1);
00622     }
00623     cpl_frame_delete(catindex);
00624 
00625     /* Is there a second photometric catalogue index file specified? */
00626 
00627     if ((catindex = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00628                                                VIRCAM_CAL_REFCAT)) != NULL) {
00629     
00630         /* Get catalogue parameters */
00631 
00632         if (vircam_catpars(catindex,&(ps.catpath2),&(ps.catname2)) == VIR_FATAL) {
00633             vircam_jmp_tidy(0);
00634             cpl_frame_delete(catindex);
00635             return(-1);
00636         }
00637         cpl_frame_delete(catindex);
00638     }
00639 
00640     /* Get the number of DITs */
00641 
00642     pp = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.science_frames,0)),0);
00643     if (vircam_pfits_get_ndit(pp,&ndit) != VIR_OK) {
00644         cpl_msg_error(fctid,"No value for NDIT available");
00645         freepropertylist(pp);
00646         vircam_jmp_tidy(0);
00647         return(-1);
00648     }
00649     cpl_propertylist_delete(pp);
00650 
00651     /* Now, how many image extensions do we want to do? If the extension
00652        number is zero, then we loop for all possible extensions. If it
00653        isn't then we just do the extension specified */
00654 
00655     vircam_exten_range(vircam_jmp_config.extenum,
00656                        (const cpl_frame *)cpl_frameset_get_frame(ps.science_frames,0),
00657                        &jst,&jfn);
00658     if (jst == -1 || jfn == -1) {
00659         cpl_msg_error(fctid,"Unable to continue");
00660         vircam_jmp_tidy(0);
00661         return(-1);
00662     }
00663 
00664     /* Now loop for all the extensions... */
00665 
00666     status = VIR_OK;
00667     for (j = jst; j <= jfn; j++) {
00668         isfirst = (j == jst);
00669         gaincor_fac = (ps.gaincors)[j-1];
00670 
00671         /* Load up the calibration frames into vir_fits/vir_tfits structures 
00672            It is a fatal error if any one of them can't load properly */
00673         
00674         ps.fdark = vircam_fits_load(ps.master_dark,CPL_TYPE_FLOAT,j);
00675         if (ps.fdark == NULL) {
00676             cpl_msg_error(fctid,
00677                           "Error loading master dark %s[%" CPL_SIZE_FORMAT "]\n%s",
00678                           cpl_frame_get_filename(ps.master_dark),(cpl_size)j,
00679                           cpl_error_get_message());
00680             vircam_jmp_tidy(0);
00681             return(-1);
00682         }
00683         ps.fflat = vircam_fits_load(ps.master_twilight_flat,CPL_TYPE_FLOAT,j);
00684         if (ps.fflat == NULL) {
00685             cpl_msg_error(fctid,
00686                           "Error loading master flat %s[%" CPL_SIZE_FORMAT "]\n%s",
00687                           cpl_frame_get_filename(ps.master_twilight_flat),
00688                           (cpl_size)j,cpl_error_get_message());
00689             vircam_jmp_tidy(0);
00690             return(-1);
00691         }
00692         ps.fconf = vircam_fits_load(ps.master_conf,CPL_TYPE_INT,j);
00693         if (ps.fconf == NULL) {
00694             cpl_msg_error(fctid,
00695                           "Error loading master conf %s[%" CPL_SIZE_FORMAT "]\n%s",
00696                           cpl_frame_get_filename(ps.master_conf),(cpl_size)j,
00697                           cpl_error_get_message());
00698             vircam_jmp_tidy(0);
00699             return(-1);
00700         }
00701         if (! isconf) 
00702             vircam_jmp_bpm2conf();
00703         if (vircam_mask_load(ps.mask,j,
00704                              (int)cpl_image_get_size_x(vircam_fits_get_image(ps.fconf)),
00705                              (int)cpl_image_get_size_y(vircam_fits_get_image(ps.fconf))) != VIR_OK) {
00706             cpl_msg_error(fctid,
00707                           "Error loading mask from master conf %s[%" CPL_SIZE_FORMAT "]\n%s",
00708                           cpl_frame_get_filename(ps.master_conf),(cpl_size)j,
00709                           cpl_error_get_message());
00710             vircam_jmp_tidy(0);
00711             return(-1);
00712         }
00713         ps.fchantab = vircam_tfits_load(ps.chantab,j);
00714         if (ps.fchantab == NULL) {
00715             cpl_msg_error(fctid,
00716                           "Error loading channel table %s[%" CPL_SIZE_FORMAT "]\n%s",
00717                           cpl_frame_get_filename(ps.chantab),(cpl_size)j,
00718                           cpl_error_get_message());
00719             vircam_jmp_tidy(0);
00720             return(-1);
00721         }
00722 
00723         /* Load up the vir_fits structures for the science images */
00724 
00725         ps.nscience = cpl_frameset_get_size(ps.science_frames);
00726         ps.sci_fits = vircam_fits_load_list(ps.science_frames,CPL_TYPE_FLOAT,j);
00727         if (ps.sci_fits == NULL) {
00728             cpl_msg_error(fctid,
00729                           "Error loading science frames extension %" CPL_SIZE_FORMAT ": %s",
00730                           (cpl_size)j,cpl_error_get_message());
00731             vircam_jmp_tidy(0);
00732             return(-1);
00733         }
00734 
00735         /* Set up something for the PAFs */
00736 
00737         ps.phupaf = vircam_paf_phu_items(vircam_fits_get_phu(ps.sci_fits[0]));
00738 
00739         /* Loop through and mark the frames where the header says the detector
00740            wasn't live */
00741 
00742         for (i = 0; i < ps.nscience; i++) {
00743             ff = ps.sci_fits[i];
00744             vircam_pfits_get_detlive(vircam_fits_get_ehu(ff),&live);
00745             if (! live) 
00746                 vircam_fits_set_error(ff,VIR_FATAL);
00747             if (vircam_check_crval(vircam_fits_get_phu(ff),
00748                                    vircam_fits_get_ehu(ff)) != VIR_OK) {
00749                 cpl_msg_error(fctid,"Unable to correct CRVAL in %s",
00750                               vircam_fits_get_fullname(ff));
00751                 vircam_jmp_tidy(0);
00752                 return(-1);
00753             }
00754         }
00755 
00756         /* Get the readnoise and gain estimate for this extension */
00757 
00758         vircam_jmp_get_readnoise_gain(j,&readnoise,&gain);
00759 
00760         /* Loop for all the science frames and do the 2d corrections */
00761 
00762         cpl_msg_info(fctid,"Doing stage1 corrections on %s",
00763                      vircam_fits_get_extname(ps.sci_fits[0]));
00764         for (i = 0; i < ps.nscience; i++) {
00765             ff = ps.sci_fits[i];
00766             cpl_propertylist_update_float(vircam_fits_get_ehu(ff),"READNOIS",
00767                                           readnoise);
00768             cpl_propertylist_set_comment(vircam_fits_get_ehu(ff),"READNOIS",
00769                                          "[e-] Readnoise used in processing");
00770             cpl_propertylist_update_float(vircam_fits_get_ehu(ff),"GAIN",gain);
00771             cpl_propertylist_set_comment(vircam_fits_get_ehu(ff),"GAIN",
00772                                          "[e-/adu] Gain used in processing");
00773             if (vircam_fits_get_status(ff) == VIR_FATAL) {
00774                 cpl_msg_info(fctid,"Detector is flagged dead in %s",
00775                              vircam_fits_get_fullname(ff));
00776                 continue;
00777             }
00778             status = VIR_OK;
00779             (void)vircam_darkcor(ff,ps.fdark,1.0,&status);
00780             (void)vircam_lincor(ff,ps.fchantab,1,ndit,&status);
00781             (void)vircam_nditcor(ff,ndit,&status);
00782             (void)vircam_flatcor(ff,ps.fflat,&status);
00783             (void)vircam_gaincor(ff,gaincor_fac,&status);
00784             vircam_fits_set_error(ff,status);
00785         }
00786 
00787         /* Do a simple sky correction if requested */
00788 
00789         if (vircam_jmp_config.skycor) {
00790             cpl_msg_info(fctid,"Doing sky correction");
00791             vircam_jmp_skycor();
00792         } else {
00793             for (i = 0; i < ps.nscience; i++) {
00794                 pp = vircam_fits_get_ehu(ps.sci_fits[i]);
00795                 cpl_propertylist_update_string(pp,"ESO DRS SKYCOR","none");
00796                 cpl_propertylist_set_comment(pp,"ESO DRS SKYCOR",
00797                                              "Sky correction method");
00798             }
00799         }
00800 
00801         /* Do destripe now */
00802 
00803         if (vircam_jmp_config.destripe) {
00804             cpl_msg_info(fctid,"Doing destripe");
00805             for (i = 0; i < ps.nscience; i++) {
00806                 ff = ps.sci_fits[i];
00807                 (void)vircam_destripe(ff,ps.mask,&status);
00808                 vircam_fits_set_error(ff,status);
00809             }           
00810         }
00811 
00812         /* Calculate the illumination correction */
00813 
00814         cpl_msg_info(fctid,"Doing illumination correction");
00815         (void)strcpy(current_cat,ps.catname);
00816         (void)strcpy(current_catpath,ps.catpath);
00817         vircam_jmp_illum();     
00818         
00819         /* Save the simple images */ 
00820 
00821         if (vircam_jmp_config.savesimple) {
00822             cpl_msg_info(fctid,"Saving simple images");
00823             if (vircam_jmp_save_simple(framelist,parlist) != 0) {
00824                 vircam_jmp_tidy(0);
00825                 return(-1);
00826             }
00827         }
00828         vircam_mask_clear(ps.mask);
00829 
00830         /* Save the illumination correction table */
00831 
00832         cpl_msg_info(fctid,"Saving illumination correction table");
00833         dummyqc = vircam_standard_process_dummyqc(3);
00834         if (vircam_jmp_save_illum(framelist,parlist) != 0) {
00835             vircam_jmp_tidy(0);
00836             freepropertylist(dummyqc);
00837             return(-1);
00838         }
00839         freepropertylist(dummyqc);
00840 
00841         /* Look at the first frame in the list and see if the number of
00842            microsteps is greater than 1. If so, then we'll have to go
00843            through interleaving. */
00844 
00845         retval = vircam_pfits_get_nusteps(vircam_fits_get_phu(ps.sci_fits[0]),
00846                                           &nusteps);
00847         if (retval != VIR_OK) {
00848             cpl_msg_warning(fctid,"Unable to get nusteps from header.\nAssuming no microstepping");
00849             nusteps = 1;
00850         }
00851         
00852         /* If the number of microsteps is 1 then copy over the good frames
00853            into a fits list for dithering. */
00854 
00855         ps.ndith = 0;
00856         if (nusteps < 4) {
00857             if (nusteps == 1) 
00858                 cpl_msg_info(fctid,"No interleaving will be done");
00859             else
00860                 cpl_msg_warning(fctid,
00861                                 "Illegal number of microsteps: %" CPL_SIZE_FORMAT "\nNo interleaving will be done",
00862                                 (cpl_size)nusteps);
00863             ps.dith_input = cpl_malloc(ps.nscience*sizeof(vir_fits *));
00864             ps.dithc_input = cpl_malloc(sizeof(vir_fits *));
00865             for (i = 0; i < ps.nscience; i++) {
00866                 if (vircam_fits_get_status(ps.sci_fits[i]) == VIR_OK) 
00867                     ps.dith_input[ps.ndith++] = ps.sci_fits[i];
00868             }
00869             ps.dithc_input[0] = ps.fconf;
00870             ps.ndithc = 1;
00871             interlv = 0;
00872 
00873         /* If the number of microsteps is more than 1, then we need
00874            to do interleaving. The interleaving routine define
00875            ps.dith_input. */
00876 
00877         } else {
00878             cpl_msg_info(fctid,"Interleaving");
00879             vircam_jmp_interleave();
00880             cpl_msg_info(fctid,"Saving superframe images");
00881             if (vircam_jmp_save_super(framelist,parlist) != 0) {
00882                 vircam_jmp_tidy(0);
00883                 return(-1);
00884             }
00885             interlv = 1;
00886         }
00887 
00888         /* Work out the jitter offsets and the stack the jitter frame */
00889 
00890         cpl_msg_info(fctid,"Working out jitter offsets");
00891         vircam_jmp_dither_offsets();
00892         cpl_msg_info(fctid,"Stacking jittered frame");
00893         vircam_jmp_dither_images();
00894 
00895         /* Do a catalogue generation */
00896 
00897         cpl_msg_info(fctid,"Doing object extraction");
00898         vircam_jmp_catalogue();
00899 
00900         /* Create a matched standards table */
00901 
00902         cpl_msg_info(fctid,"Matching objects with 2mass standards");
00903         vircam_jmp_matched_stds();
00904 
00905         /* Do a WCS fit for the dithered image */
00906 
00907         cpl_msg_info(fctid,"Fitting a WCS");
00908         vircam_jmp_wcsfit();
00909 
00910         /* Finally do the photometric zeropoint fit */
00911 
00912         cpl_msg_info(fctid,"Doing 2mass photometric zeropoint calculation");
00913         vircam_jmp_photcal();
00914         if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00915             ehu = vircam_fits_get_ehu(ps.stack_frame);
00916             cpl_propertylist_update_float(ehu,"ESO QC ZPT_2MASS",
00917                                           cpl_propertylist_get_float(ehu,"ESO QC MAGZPT"));
00918             cpl_propertylist_set_comment(ehu,"ESO QC ZPT_2MASS",
00919                                          cpl_propertylist_get_comment(ehu,"ESO QC MAGZPT"));
00920         }
00921 
00922         /* Redo the photometric zeropoint with another catalogue if we
00923            have one */
00924 
00925         if (ps.catpath2 != NULL) {
00926             freetable(ps.matchstds);
00927             (void)strcpy(current_cat,ps.catname2);
00928             (void)strcpy(current_catpath,ps.catpath2);
00929             cpl_msg_info(fctid,"Matching objects with %s standards",
00930                          ps.catname2);
00931             vircam_jmp_matched_stds();
00932             cpl_msg_info(fctid,"Doing %s photometric zeropoint calculation",
00933                          ps.catname2);
00934             vircam_jmp_photcal();
00935             if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00936                 ehu = vircam_fits_get_ehu(ps.stack_frame);
00937                 cpl_propertylist_update_float(ehu,"ESO QC ZPT_STDS",
00938                                               cpl_propertylist_get_float(ehu,"ESO QC MAGZPT"));
00939                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS",
00940                                              cpl_propertylist_get_comment(ehu,"ESO QC MAGZPT"));
00941                 cpl_propertylist_update_string(ehu,"ESO QC ZPT_STDS_CAT",
00942                                                ps.catname2);
00943                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS_CAT",
00944                                              "Catalogue used in zeropoint calc");
00945             }
00946         } else {
00947             if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00948                 ehu = vircam_fits_get_ehu(ps.stack_frame);
00949                 cpl_propertylist_update_float(ehu,"ESO QC ZPT_STDS",0.0);
00950                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS",
00951                                              "No secondary standard calib");
00952                 cpl_propertylist_update_string(ehu,"ESO QC ZPT_STDS_CAT","");
00953                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS_CAT",
00954                                              "No secondary standard catalogue");
00955             }
00956         }
00957 
00958         /* Save the dithered images */
00959 
00960         cpl_msg_info(fctid,"Saving stacked images");    
00961         dummyqc = vircam_standard_process_dummyqc(1);
00962         if (vircam_jmp_save_stack(framelist,parlist) != 0) {
00963             vircam_jmp_tidy(0);
00964             freepropertylist(dummyqc);
00965             return(-1);
00966         }
00967         freepropertylist(dummyqc);
00968         if (vircam_jmp_config.savecat) {
00969             cpl_msg_info(fctid,"Saving stacked image catalogues");
00970             dummyqc = vircam_standard_process_dummyqc(2);
00971             if (vircam_jmp_save_catalogue(framelist,parlist) != 0) {
00972                 vircam_jmp_tidy(0);
00973                 freepropertylist(dummyqc);
00974                 return(-1);
00975             }
00976             freepropertylist(dummyqc);
00977         }
00978 
00979         /* Clean up on aisle 12! */
00980 
00981         vircam_jmp_tidy(1);
00982     }
00983 
00984     /* Final cleanup */
00985 
00986     vircam_jmp_tidy(0);
00987     return(0);
00988 }
00989 
00990 static cpl_propertylist *vircam_standard_process_dummyqc(int type) {
00991     cpl_propertylist *p;
00992 
00993     /* Get an empty property list */
00994 
00995     p = cpl_propertylist_new();
00996 
00997     /* Now switch for the various products */
00998 
00999     switch (type) {
01000 
01001     /* Stack images */
01002 
01003     case 1:
01004         cpl_propertylist_update_double(p,"ESO QC WCS_DCRVAL1",0.0);
01005         cpl_propertylist_set_comment(p,"ESO QC WCS_DCRVAL1",
01006                                      "[deg] change in crval1");
01007         cpl_propertylist_update_double(p,"ESO QC WCS_DCRVAL2",0.0);
01008         cpl_propertylist_set_comment(p,"ESO QC WCS_DCRVAL2",
01009                                      "[deg] change in crval2");
01010         cpl_propertylist_update_double(p,"ESO QC WCS_DTHETA",0.0);
01011         cpl_propertylist_set_comment(p,"ESO QC WCS_DTHETA",
01012                                      "[deg] change in rotation");
01013         cpl_propertylist_update_double(p,"ESO QC WCS_SCALE",0.0);
01014         cpl_propertylist_set_comment(p,"ESO QC WCS_SCALE",
01015                                      "[arcsec] mean plate scale");
01016         cpl_propertylist_update_double(p,"ESO QC WCS_SHEAR",0.0);
01017         cpl_propertylist_set_comment(p,"ESO QC WCS_SHEAR",
01018                                      "[deg] abs(xrot) - abs(yrot)");
01019         cpl_propertylist_update_double(p,"ESO QC WCS_RMS",0.0);
01020         cpl_propertylist_set_comment(p,"ESO QC WCS_RMS",
01021                                      "[arcsec] Average error in WCS fit");
01022         cpl_propertylist_update_float(p,"ESO QC MAGZPT",0.0);
01023         cpl_propertylist_set_comment(p,"ESO QC MAGZPT",
01024                                      "[mag] photometric zeropoint");
01025         cpl_propertylist_update_float(p,"ESO QC MAGZERR",0.0);
01026         cpl_propertylist_set_comment(p,"ESO QC MAGZERR",
01027                                      "[mag] photometric zeropoint error");
01028         cpl_propertylist_update_int(p,"ESO QC MAGNZPT",0);
01029         cpl_propertylist_set_comment(p,"ESO QC MAGNZPT",
01030                                      "number of stars in magzpt calc");
01031         cpl_propertylist_update_int(p,"ESO QC MAGNCUT",0);
01032         cpl_propertylist_set_comment(p,"ESO QC MAGNCUT",
01033                                      "number of cut from magzpt calc");
01034         cpl_propertylist_update_float(p,"ESO QC LIMITING_MAG",0.0);
01035         cpl_propertylist_set_comment(p,"ESO QC LIMITING_MAG",
01036                                      "[mag] 5 sigma limiting mag");
01037         cpl_propertylist_update_float(p,"ESO QC ZPT_2MASS",0.0);
01038         cpl_propertylist_set_comment(p,"ESO QC ZPT_2MASS",
01039                                      "[mag] photometric zeropoint");
01040         cpl_propertylist_update_float(p,"ESO QC ZPT_STDS",0.0);
01041         cpl_propertylist_set_comment(p,"ESO QC ZPT_STDS",
01042                                      "[mag] photometric zeropoint");
01043         cpl_propertylist_update_string(p,"ESO QC ZPT_STDS_CAT","");
01044         cpl_propertylist_set_comment(p,"ESO QC ZPT_STDS_CAT",
01045                                      "Catalogue used in zeropoint calc");
01046         break;
01047         
01048     /* Catalogues */
01049 
01050     case 2:
01051         cpl_propertylist_update_float(p,"ESO QC SATURATION",0.0);
01052         cpl_propertylist_set_comment(p,"ESO QC SATURATION",
01053                                      "[adu] Saturation level");
01054         cpl_propertylist_update_float(p,"ESO QC MEAN_SKY",0.0);
01055         cpl_propertylist_set_comment(p,"ESO QC MEAN_SKY",
01056                                      "[adu] Median sky brightness");
01057         cpl_propertylist_update_float(p,"ESO QC SKY_NOISE",0.0);
01058         cpl_propertylist_set_comment(p,"ESO QC SKY_NOISE",
01059                                      "[adu] Pixel noise at sky level");
01060         cpl_propertylist_update_float(p,"ESO QC IMAGE_SIZE",0.0);
01061         cpl_propertylist_set_comment(p,"ESO QC IMAGE_SIZE",
01062                                      "[pixels] Average FWHM of stellar objects");
01063         cpl_propertylist_update_float(p,"ESO QC ELLIPTICITY",0.0);
01064         cpl_propertylist_set_comment(p,"ESO QC ELLIPTICITY",
01065                                      "Average stellar ellipticity (1-b/a)");
01066         cpl_propertylist_update_float(p,"ESO QC APERTURE_CORR",0.0);
01067         cpl_propertylist_set_comment(p,"ESO QC APERTURE_CORR",
01068                                      "Stellar ap-corr 1x core flux");
01069         cpl_propertylist_update_int(p,"ESO QC NOISE_OBJ",0);
01070         cpl_propertylist_set_comment(p,"ESO QC NOISE_OBJ",
01071                                      "Number of noise objects");
01072         break;
01073 
01074     /* Illumination tables */
01075 
01076     case 3:
01077         cpl_propertylist_update_float(p,"ESO QC ILLUMCOR_RMS",0.0);
01078         cpl_propertylist_set_comment(p,"ESO QC ILLUMCOR_RMS",
01079                                      "RMS of illumination correction map");
01080         break;
01081     default:
01082         break;
01083     }
01084 
01085     /* Get out of here */
01086 
01087     return(p);
01088 }
01089 
01092 /*
01093 
01094 $Log: vircam_standard_process.c,v $
01095 Revision 1.43  2012/01/16 12:32:18  jim
01096 A few more changes to fit in with cpl6
01097 
01098 Revision 1.42  2012/01/15 17:40:09  jim
01099 Minor modifications to take into accout the changes in cpl API for v6
01100 
01101 Revision 1.41  2011/05/09 09:58:10  jim
01102 Cosmetic changes to stop compiler warnings
01103 
01104 Revision 1.40  2010/12/09 13:20:49  jim
01105 Modified some comments
01106 
01107 Revision 1.39  2010/09/13 11:43:13  jim
01108 Added flag DRS SKYCOR if background correction isn't being done
01109 
01110 Revision 1.38  2010/09/09 12:16:00  jim
01111 Added new QC parameter MAGNCUT
01112 
01113 Revision 1.37  2010/06/03 11:31:06  jim
01114 Modified so that deblending is done by default
01115 
01116 Revision 1.36  2010/03/21 06:47:42  jim
01117 Added code to trap for problems in saving products
01118 
01119 Revision 1.35  2010/03/12 10:44:07  lbilbao
01120 Added missing header inclusion.
01121 
01122 Revision 1.34  2010/02/08 16:35:26  jim
01123 Moved the definition of propertylist ps.phupaf to the recipes so that
01124 this doesn't fail when we don't save the simple images
01125 
01126 Revision 1.33  2010/01/31 18:56:35  jim
01127 Now optionally saves the simple images. Also included missing ndit correction
01128 
01129 Revision 1.32  2009/12/11 06:52:56  jim
01130 Minor mods to documentation
01131 
01132 Revision 1.31  2009/09/09 09:50:21  jim
01133 Modified to try and get headers right
01134 
01135 Revision 1.30  2009/07/03 12:30:04  jim
01136 Default value of rcore is now 3
01137 
01138 Revision 1.29  2009/02/04 09:23:29  jim
01139 Moved destriping to after sky subtraction
01140 
01141 Revision 1.28  2008/12/08 06:38:41  jim
01142 Added trap for crval==0 error. Also added a trap for illegal 2x1 microstep
01143 sequences
01144 
01145 Revision 1.27  2008/11/27 09:14:47  jim
01146 Fixed PRO CATG keyword values for interleaved confidence maps in docs
01147 
01148 Revision 1.26  2008/11/25 18:55:59  jim
01149 took out extra sky stuff
01150 
01151 Revision 1.25  2008/11/25 12:03:03  jim
01152 Now allows for masked object sky estimation
01153 
01154 Revision 1.24  2008/11/25 06:18:30  jim
01155 Changed a reference to VIRCAM_CAL_2MASS to VIRCAM_CAL_REFCAT.
01156 
01157 Revision 1.23  2008/10/01 04:59:13  jim
01158 Added call to vircam_frameset_fexists to check input frameset
01159 
01160 Revision 1.22  2008/05/06 12:15:21  jim
01161 Changed to use new version of vircam_catpars
01162 
01163 Revision 1.21  2007/11/26 09:59:06  jim
01164 Recipe now takes ndit into account when doing linearity correction
01165 
01166 Revision 1.20  2007/10/25 18:39:22  jim
01167 Altered to remove some lint messages
01168 
01169 Revision 1.19  2007/10/19 06:55:06  jim
01170 Modifications made to use new method for directing the recipes to the
01171 standard catalogues using the sof
01172 
01173 Revision 1.18  2007/07/09 13:21:56  jim
01174 Modified to use new version of vircam_exten_range
01175 
01176 Revision 1.17  2007/06/13 08:11:27  jim
01177 Modified docs to reflect changes in DFS tags
01178 
01179 Revision 1.16  2007/05/08 21:31:16  jim
01180 fixed typo
01181 
01182 Revision 1.15  2007/05/08 10:42:44  jim
01183 Added gain correction
01184 
01185 Revision 1.14  2007/05/02 12:53:11  jim
01186 typo fixes in docs
01187 
01188 Revision 1.13  2007/04/26 13:09:40  jim
01189 fixed typos
01190 
01191 Revision 1.12  2007/04/04 16:05:59  jim
01192 Modified to make paf information a bit more correct
01193 
01194 Revision 1.11  2007/04/04 10:36:18  jim
01195 Modified to use new dfs tags
01196 
01197 Revision 1.10  2007/03/29 12:19:39  jim
01198 Little changes to improve documentation
01199 
01200 Revision 1.9  2007/03/14 14:49:13  jim
01201 Fixed problem with missing paf files in jmp recipes if detlive = F. Also
01202 fixed problem where extra dummy products were being created
01203 
01204 Revision 1.8  2007/03/06 12:00:48  jim
01205 Fixed stupid typo in header
01206 
01207 Revision 1.7  2007/03/02 12:37:51  jim
01208 Fixed small memory leak
01209 
01210 Revision 1.6  2007/03/01 12:41:49  jim
01211 Modified slightly after code checking
01212 
01213 Revision 1.5  2007/02/07 10:12:40  jim
01214 Removed calls to vircam_ndit_correct as this is now no longer necessary
01215 
01216 Revision 1.4  2006/12/19 13:32:03  jim
01217 Images that are flagged as dead detectors now generate an INFO rather than
01218 an ERROR message
01219 
01220 Revision 1.3  2006/11/29 12:28:45  jim
01221 Modified so that the correct recipe names would appear in the headers of
01222 data products
01223 
01224 Revision 1.2  2006/11/28 20:57:43  jim
01225 Added illumination correction section
01226 
01227 Revision 1.1  2006/11/27 12:15:43  jim
01228 Initial entry
01229 
01230 
01231 */

Generated on 5 Mar 2013 for VIRCAM Pipeline by  doxygen 1.6.1