39 #include "vircam_utils.h"
40 #include "vircam_mask.h"
41 #include "vircam_pfits.h"
42 #include "vircam_dfs.h"
43 #include "vircam_mods.h"
44 #include "vircam_fits.h"
45 #include "vircam_tfits.h"
46 #include "vircam_jmp_utils.h"
47 #include "vircam_paf.h"
51 static int vircam_standard_process_create(cpl_plugin *);
52 static int vircam_standard_process_exec(cpl_plugin *);
53 static int vircam_standard_process_destroy(cpl_plugin *);
54 static int vircam_standard_process(cpl_parameterlist *,cpl_frameset *);
55 static cpl_propertylist *vircam_standard_process_dummyqc(
int type);
58 static char vircam_standard_process_description[] =
59 "vircam_standard_process -- VIRCAM standard field processing recipe.\n\n"
60 "Process a complete pawprint for standard fields in VIRCAM data.\n"
61 "Remove instrumental signature, interleave microstep sequences (if done),\n"
62 "combine jitters, photometrically and astrometrically calibrate the pawprint\n"
63 "image. Photometric calibration can be done with 2mass and 1 other source\n\n"
64 "The program accepts the following files in the SOF:\n\n"
66 " -----------------------------------------------------------------------\n"
67 " %-21s A list of raw science images\n"
68 " %-21s A master dark frame\n"
69 " %-21s A master twilight flat frame\n"
70 " %-21s A channel table\n"
71 " %-21s A photometric calibration table\n"
72 " %-21s A readnoise/gain file\n"
73 " %-21s A master confidence map or\n"
74 " %-21s A master bad pixel mask\n"
75 " %-21s A master 2mass catalogue index\n"
76 " %-21s A second reference catalogue index\n"
77 "All of the above are required\n"
235 int cpl_plugin_get_info(cpl_pluginlist *list) {
236 cpl_recipe *recipe = cpl_calloc(1,
sizeof(*recipe));
237 cpl_plugin *plugin = &recipe->interface;
238 char alldesc[SZ_ALLDESC];
239 (void)snprintf(alldesc,SZ_ALLDESC,vircam_standard_process_description,
240 VIRCAM_STD_OBJECT_RAW,VIRCAM_CAL_DARK,
241 VIRCAM_CAL_TWILIGHT_FLAT,VIRCAM_CAL_CHANTAB,
242 VIRCAM_CAL_PHOTTAB,VIRCAM_CAL_READGAINFILE,VIRCAM_CAL_CONF,
243 VIRCAM_CAL_BPM,VIRCAM_CAL_2MASS,VIRCAM_CAL_REFCAT);
245 cpl_plugin_init(plugin,
247 VIRCAM_BINARY_VERSION,
248 CPL_PLUGIN_TYPE_RECIPE,
249 "vircam_standard_process",
250 "VIRCAM standard field processing recipe",
255 vircam_standard_process_create,
256 vircam_standard_process_exec,
257 vircam_standard_process_destroy);
259 cpl_pluginlist_append(list,plugin);
275 static int vircam_standard_process_create(cpl_plugin *plugin) {
281 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
282 recipe = (cpl_recipe *)plugin;
288 recipe->parameters = cpl_parameterlist_new();
292 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.ipix",
294 "Minimum pixel area for each detected object",
295 "vircam.vircam_standard_process",5);
296 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"ipix");
297 cpl_parameterlist_append(recipe->parameters,p);
301 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.thresh",
303 "Detection threshold in sigma above sky",
304 "vircam.vircam_standard_process",2.0);
305 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"thr");
306 cpl_parameterlist_append(recipe->parameters,p);
310 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.icrowd",
311 CPL_TYPE_BOOL,
"Use deblending?",
312 "vircam.vircam_standard_process",1);
313 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"icrowd");
314 cpl_parameterlist_append(recipe->parameters,p);
318 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.rcore",
319 CPL_TYPE_DOUBLE,
"Value of Rcore in pixels",
320 "vircam.vircam_standard_process",3.0);
321 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"rcore");
322 cpl_parameterlist_append(recipe->parameters,p);
326 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.nbsize",
327 CPL_TYPE_INT,
"Background smoothing box size",
328 "vircam.vircam_standard_process",64);
329 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"nb");
330 cpl_parameterlist_append(recipe->parameters,p);
334 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.savecat",
335 CPL_TYPE_BOOL,
"Save catalogue?",
336 "vircam.vircam_standard_process",1);
337 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"savecat");
338 cpl_parameterlist_append(recipe->parameters,p);
343 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.destripe",
344 CPL_TYPE_BOOL,
"Destripe images?",
345 "vircam.vircam_standard_process",1);
346 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"destripe");
347 cpl_parameterlist_append(recipe->parameters,p);
351 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.skycor",
352 CPL_TYPE_BOOL,
"Do sky correction?",
353 "vircam.vircam_standard_process",1);
354 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"skycor");
355 cpl_parameterlist_append(recipe->parameters,p);
359 p = cpl_parameter_new_value(
"vircam.vircam_standard_process.savesimple",
360 CPL_TYPE_BOOL,
"Save simple images?",
361 "vircam.vircam_standard_process",0);
362 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"savesimple");
363 cpl_parameterlist_append(recipe->parameters,p);
367 p = cpl_parameter_new_range(
"vircam.vircam_standard_process.extenum",
369 "Extension number to be done, 0 == all",
370 "vircam.vircam_standard_process",
372 cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,
"ext");
373 cpl_parameterlist_append(recipe->parameters,p);
389 static int vircam_standard_process_exec(cpl_plugin *plugin) {
394 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
395 recipe = (cpl_recipe *)plugin;
399 return(vircam_standard_process(recipe->parameters,recipe->frames));
410 static int vircam_standard_process_destroy(cpl_plugin *plugin) {
415 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
416 recipe = (cpl_recipe *)plugin;
420 cpl_parameterlist_delete(recipe->parameters);
433 static int vircam_standard_process(cpl_parameterlist *parlist,
434 cpl_frameset *framelist) {
435 const char *fctid=
"vircam_standard_process";
437 int jst,jfn,status,j,i,retval,nusteps,isconf,live,ndit;
439 float readnoise,gain,gaincor_fac;
441 cpl_propertylist *ehu,*pp;
446 if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
447 cpl_msg_error(fctid,
"Input framelist NULL or has no input data");
454 cpl_msg_error(fctid,
"Input frameset is missing files. Check SOF");
461 (void)strncpy(vircam_recipename,fctid,VIRCAM_PATHSZ);
462 (void)snprintf(vircam_recipepaf,VIRCAM_PATHSZ,
"VIRCAM/%s",fctid);
467 p = cpl_parameterlist_find(parlist,
468 "vircam.vircam_standard_process.ipix");
469 vircam_jmp_config.ipix = cpl_parameter_get_int(p);
470 p = cpl_parameterlist_find(parlist,
471 "vircam.vircam_standard_process.thresh");
472 vircam_jmp_config.threshold = (float)cpl_parameter_get_double(p);
473 p = cpl_parameterlist_find(parlist,
474 "vircam.vircam_standard_process.icrowd");
475 vircam_jmp_config.icrowd = cpl_parameter_get_bool(p);
476 p = cpl_parameterlist_find(parlist,
477 "vircam.vircam_standard_process.rcore");
478 vircam_jmp_config.rcore = (float)cpl_parameter_get_double(p);
479 p = cpl_parameterlist_find(parlist,
480 "vircam.vircam_standard_process.nbsize");
481 vircam_jmp_config.nbsize = cpl_parameter_get_int(p);
482 p = cpl_parameterlist_find(parlist,
483 "vircam.vircam_standard_process.savecat");
484 vircam_jmp_config.savecat = cpl_parameter_get_bool(p);
485 p = cpl_parameterlist_find(parlist,
486 "vircam.vircam_standard_process.destripe");
487 vircam_jmp_config.destripe = cpl_parameter_get_bool(p);
488 p = cpl_parameterlist_find(parlist,
489 "vircam.vircam_standard_process.skycor");
490 vircam_jmp_config.skycor = cpl_parameter_get_bool(p);
491 p = cpl_parameterlist_find(parlist,
492 "vircam.vircam_standard_process.savesimple");
493 vircam_jmp_config.savesimple = cpl_parameter_get_bool(p);
494 p = cpl_parameterlist_find(parlist,
495 "vircam.vircam_standard_process.extenum");
496 vircam_jmp_config.extenum = cpl_parameter_get_int(p);
501 cpl_msg_error(fctid,
"Cannot identify RAW and CALIB frames");
510 cpl_msg_error(fctid,
"Cannot labelise the input frames");
517 if ((ps.science_frames =
519 VIRCAM_STD_OBJECT_RAW)) == NULL) {
520 cpl_msg_error(fctid,
"No science images to process!");
527 if ((ps.master_dark =
529 VIRCAM_CAL_DARK)) == NULL) {
530 cpl_msg_error(fctid,
"No master dark found");
537 if ((ps.master_twilight_flat =
539 VIRCAM_CAL_TWILIGHT_FLAT)) == NULL) {
540 cpl_msg_error(fctid,
"No master twilight flat found");
549 &status) != VIR_OK) {
550 cpl_msg_error(fctid,
"Error calculating gain corrections");
557 if ((ps.readgain_file =
559 VIRCAM_CAL_READGAINFILE)) == NULL) {
560 cpl_msg_error(fctid,
"No master readnoise/gain file found");
570 if ((ps.master_conf =
572 VIRCAM_CAL_CONF)) == NULL) {
574 if ((ps.master_conf =
576 VIRCAM_CAL_BPM)) == NULL) {
577 cpl_msg_error(fctid,
"No master confidence map found");
587 VIRCAM_CAL_CHANTAB)) == NULL) {
588 cpl_msg_error(fctid,
"No channel table found");
596 VIRCAM_CAL_PHOTTAB)) == NULL) {
597 cpl_msg_error(fctid,
"No photometric table found");
601 if ((ps.tphottab = cpl_table_load(cpl_frame_get_filename(ps.phottab),1,0)) == NULL) {
602 cpl_msg_error(fctid,
"Unable to load photometric table");
610 VIRCAM_CAL_2MASS)) == NULL) {
611 cpl_msg_info(fctid,
"No 2MASS index found -- cannot continue");
618 if (
vircam_catpars(catindex,&(ps.catpath),&(ps.catname)) == VIR_FATAL) {
620 cpl_frame_delete(catindex);
623 cpl_frame_delete(catindex);
628 VIRCAM_CAL_REFCAT)) != NULL) {
632 if (
vircam_catpars(catindex,&(ps.catpath2),&(ps.catname2)) == VIR_FATAL) {
634 cpl_frame_delete(catindex);
637 cpl_frame_delete(catindex);
642 pp = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.science_frames,0)),0);
644 cpl_msg_error(fctid,
"No value for NDIT available");
645 freepropertylist(pp);
649 cpl_propertylist_delete(pp);
656 (
const cpl_frame *)cpl_frameset_get_frame(ps.science_frames,0),
658 if (jst == -1 || jfn == -1) {
659 cpl_msg_error(fctid,
"Unable to continue");
667 for (j = jst; j <= jfn; j++) {
668 isfirst = (j == jst);
669 gaincor_fac = (ps.gaincors)[j-1];
675 if (ps.fdark == NULL) {
677 "Error loading master dark %s[%" CPL_SIZE_FORMAT
"]\n%s",
678 cpl_frame_get_filename(ps.master_dark),(cpl_size)j,
679 cpl_error_get_message());
684 if (ps.fflat == NULL) {
686 "Error loading master flat %s[%" CPL_SIZE_FORMAT
"]\n%s",
687 cpl_frame_get_filename(ps.master_twilight_flat),
688 (cpl_size)j,cpl_error_get_message());
693 if (ps.fconf == NULL) {
695 "Error loading master conf %s[%" CPL_SIZE_FORMAT
"]\n%s",
696 cpl_frame_get_filename(ps.master_conf),(cpl_size)j,
697 cpl_error_get_message());
707 "Error loading mask from master conf %s[%" CPL_SIZE_FORMAT
"]\n%s",
708 cpl_frame_get_filename(ps.master_conf),(cpl_size)j,
709 cpl_error_get_message());
714 if (ps.fchantab == NULL) {
716 "Error loading channel table %s[%" CPL_SIZE_FORMAT
"]\n%s",
717 cpl_frame_get_filename(ps.chantab),(cpl_size)j,
718 cpl_error_get_message());
725 ps.nscience = cpl_frameset_get_size(ps.science_frames);
727 if (ps.sci_fits == NULL) {
729 "Error loading science frames extension %" CPL_SIZE_FORMAT
": %s",
730 (cpl_size)j,cpl_error_get_message());
742 for (i = 0; i < ps.nscience; i++) {
749 cpl_msg_error(fctid,
"Unable to correct CRVAL in %s",
762 cpl_msg_info(fctid,
"Doing stage1 corrections on %s",
764 for (i = 0; i < ps.nscience; i++) {
769 "[e-] Readnoise used in processing");
772 "[e-/adu] Gain used in processing");
774 cpl_msg_info(fctid,
"Detector is flagged dead in %s",
789 if (vircam_jmp_config.skycor) {
790 cpl_msg_info(fctid,
"Doing sky correction");
793 for (i = 0; i < ps.nscience; i++) {
795 cpl_propertylist_update_string(pp,
"ESO DRS SKYCOR",
"none");
796 cpl_propertylist_set_comment(pp,
"ESO DRS SKYCOR",
797 "Sky correction method");
803 if (vircam_jmp_config.destripe) {
804 cpl_msg_info(fctid,
"Doing destripe");
805 for (i = 0; i < ps.nscience; i++) {
814 cpl_msg_info(fctid,
"Doing illumination correction");
815 (void)strcpy(current_cat,ps.catname);
816 (void)strcpy(current_catpath,ps.catpath);
821 if (vircam_jmp_config.savesimple) {
822 cpl_msg_info(fctid,
"Saving simple images");
832 cpl_msg_info(fctid,
"Saving illumination correction table");
833 dummyqc = vircam_standard_process_dummyqc(3);
836 freepropertylist(dummyqc);
839 freepropertylist(dummyqc);
847 if (retval != VIR_OK) {
848 cpl_msg_warning(fctid,
"Unable to get nusteps from header.\nAssuming no microstepping");
858 cpl_msg_info(fctid,
"No interleaving will be done");
860 cpl_msg_warning(fctid,
861 "Illegal number of microsteps: %" CPL_SIZE_FORMAT
"\nNo interleaving will be done",
863 ps.dith_input = cpl_malloc(ps.nscience*
sizeof(vir_fits *));
864 ps.dithc_input = cpl_malloc(
sizeof(vir_fits *));
865 for (i = 0; i < ps.nscience; i++) {
867 ps.dith_input[ps.ndith++] = ps.sci_fits[i];
869 ps.dithc_input[0] = ps.fconf;
878 cpl_msg_info(fctid,
"Interleaving");
880 cpl_msg_info(fctid,
"Saving superframe images");
890 cpl_msg_info(fctid,
"Working out jitter offsets");
892 cpl_msg_info(fctid,
"Stacking jittered frame");
897 cpl_msg_info(fctid,
"Doing object extraction");
902 cpl_msg_info(fctid,
"Matching objects with 2mass standards");
907 cpl_msg_info(fctid,
"Fitting a WCS");
912 cpl_msg_info(fctid,
"Doing 2mass photometric zeropoint calculation");
916 cpl_propertylist_update_float(ehu,
"ESO QC ZPT_2MASS",
917 cpl_propertylist_get_float(ehu,
"ESO QC MAGZPT"));
918 cpl_propertylist_set_comment(ehu,
"ESO QC ZPT_2MASS",
919 cpl_propertylist_get_comment(ehu,
"ESO QC MAGZPT"));
925 if (ps.catpath2 != NULL) {
926 freetable(ps.matchstds);
927 (void)strcpy(current_cat,ps.catname2);
928 (void)strcpy(current_catpath,ps.catpath2);
929 cpl_msg_info(fctid,
"Matching objects with %s standards",
932 cpl_msg_info(fctid,
"Doing %s photometric zeropoint calculation",
937 cpl_propertylist_update_float(ehu,
"ESO QC ZPT_STDS",
938 cpl_propertylist_get_float(ehu,
"ESO QC MAGZPT"));
939 cpl_propertylist_set_comment(ehu,
"ESO QC ZPT_STDS",
940 cpl_propertylist_get_comment(ehu,
"ESO QC MAGZPT"));
941 cpl_propertylist_update_string(ehu,
"ESO QC ZPT_STDS_CAT",
943 cpl_propertylist_set_comment(ehu,
"ESO QC ZPT_STDS_CAT",
944 "Catalogue used in zeropoint calc");
949 cpl_propertylist_update_float(ehu,
"ESO QC ZPT_STDS",0.0);
950 cpl_propertylist_set_comment(ehu,
"ESO QC ZPT_STDS",
951 "No secondary standard calib");
952 cpl_propertylist_update_string(ehu,
"ESO QC ZPT_STDS_CAT",
"");
953 cpl_propertylist_set_comment(ehu,
"ESO QC ZPT_STDS_CAT",
954 "No secondary standard catalogue");
960 cpl_msg_info(fctid,
"Saving stacked images");
961 dummyqc = vircam_standard_process_dummyqc(1);
964 freepropertylist(dummyqc);
967 freepropertylist(dummyqc);
968 if (vircam_jmp_config.savecat) {
969 cpl_msg_info(fctid,
"Saving stacked image catalogues");
970 dummyqc = vircam_standard_process_dummyqc(2);
973 freepropertylist(dummyqc);
976 freepropertylist(dummyqc);
990 static cpl_propertylist *vircam_standard_process_dummyqc(
int type) {
995 p = cpl_propertylist_new();
1004 cpl_propertylist_update_double(p,
"ESO QC WCS_DCRVAL1",0.0);
1005 cpl_propertylist_set_comment(p,
"ESO QC WCS_DCRVAL1",
1006 "[deg] change in crval1");
1007 cpl_propertylist_update_double(p,
"ESO QC WCS_DCRVAL2",0.0);
1008 cpl_propertylist_set_comment(p,
"ESO QC WCS_DCRVAL2",
1009 "[deg] change in crval2");
1010 cpl_propertylist_update_double(p,
"ESO QC WCS_DTHETA",0.0);
1011 cpl_propertylist_set_comment(p,
"ESO QC WCS_DTHETA",
1012 "[deg] change in rotation");
1013 cpl_propertylist_update_double(p,
"ESO QC WCS_SCALE",0.0);
1014 cpl_propertylist_set_comment(p,
"ESO QC WCS_SCALE",
1015 "[arcsec] mean plate scale");
1016 cpl_propertylist_update_double(p,
"ESO QC WCS_SHEAR",0.0);
1017 cpl_propertylist_set_comment(p,
"ESO QC WCS_SHEAR",
1018 "[deg] abs(xrot) - abs(yrot)");
1019 cpl_propertylist_update_double(p,
"ESO QC WCS_RMS",0.0);
1020 cpl_propertylist_set_comment(p,
"ESO QC WCS_RMS",
1021 "[arcsec] Average error in WCS fit");
1022 cpl_propertylist_update_float(p,
"ESO QC MAGZPT",0.0);
1023 cpl_propertylist_set_comment(p,
"ESO QC MAGZPT",
1024 "[mag] photometric zeropoint");
1025 cpl_propertylist_update_float(p,
"ESO QC MAGZERR",0.0);
1026 cpl_propertylist_set_comment(p,
"ESO QC MAGZERR",
1027 "[mag] photometric zeropoint error");
1028 cpl_propertylist_update_int(p,
"ESO QC MAGNZPT",0);
1029 cpl_propertylist_set_comment(p,
"ESO QC MAGNZPT",
1030 "number of stars in magzpt calc");
1031 cpl_propertylist_update_int(p,
"ESO QC MAGNCUT",0);
1032 cpl_propertylist_set_comment(p,
"ESO QC MAGNCUT",
1033 "number of cut from magzpt calc");
1034 cpl_propertylist_update_float(p,
"ESO QC LIMITING_MAG",0.0);
1035 cpl_propertylist_set_comment(p,
"ESO QC LIMITING_MAG",
1036 "[mag] 5 sigma limiting mag");
1037 cpl_propertylist_update_float(p,
"ESO QC ZPT_2MASS",0.0);
1038 cpl_propertylist_set_comment(p,
"ESO QC ZPT_2MASS",
1039 "[mag] photometric zeropoint");
1040 cpl_propertylist_update_float(p,
"ESO QC ZPT_STDS",0.0);
1041 cpl_propertylist_set_comment(p,
"ESO QC ZPT_STDS",
1042 "[mag] photometric zeropoint");
1043 cpl_propertylist_update_string(p,
"ESO QC ZPT_STDS_CAT",
"");
1044 cpl_propertylist_set_comment(p,
"ESO QC ZPT_STDS_CAT",
1045 "Catalogue used in zeropoint calc");
1051 cpl_propertylist_update_float(p,
"ESO QC SATURATION",0.0);
1052 cpl_propertylist_set_comment(p,
"ESO QC SATURATION",
1053 "[adu] Saturation level");
1054 cpl_propertylist_update_float(p,
"ESO QC MEAN_SKY",0.0);
1055 cpl_propertylist_set_comment(p,
"ESO QC MEAN_SKY",
1056 "[adu] Median sky brightness");
1057 cpl_propertylist_update_float(p,
"ESO QC SKY_NOISE",0.0);
1058 cpl_propertylist_set_comment(p,
"ESO QC SKY_NOISE",
1059 "[adu] Pixel noise at sky level");
1060 cpl_propertylist_update_float(p,
"ESO QC IMAGE_SIZE",0.0);
1061 cpl_propertylist_set_comment(p,
"ESO QC IMAGE_SIZE",
1062 "[pixels] Average FWHM of stellar objects");
1063 cpl_propertylist_update_float(p,
"ESO QC ELLIPTICITY",0.0);
1064 cpl_propertylist_set_comment(p,
"ESO QC ELLIPTICITY",
1065 "Average stellar ellipticity (1-b/a)");
1066 cpl_propertylist_update_float(p,
"ESO QC APERTURE_CORR",0.0);
1067 cpl_propertylist_set_comment(p,
"ESO QC APERTURE_CORR",
1068 "Stellar ap-corr 1x core flux");
1069 cpl_propertylist_update_int(p,
"ESO QC NOISE_OBJ",0);
1070 cpl_propertylist_set_comment(p,
"ESO QC NOISE_OBJ",
1071 "Number of noise objects");
1077 cpl_propertylist_update_float(p,
"ESO QC ILLUMCOR_RMS",0.0);
1078 cpl_propertylist_set_comment(p,
"ESO QC ILLUMCOR_RMS",
1079 "RMS of illumination correction map");