/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2007-2018 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <muse.h>

#include <math.h>
#include <string.h>

extern const muse_cpltable_def muse_filtertable_def[];

muse_table *
muse_test_flux_cousins_r(void)
{
  cpl_table *table = muse_cpltable_new(muse_filtertable_def, 24);
  cpl_table_set_double(table, "lambda",  0, 5500.);
  cpl_table_set_double(table, "lambda",  1, 5600.);
  cpl_table_set_double(table, "lambda",  2, 5700.);
  cpl_table_set_double(table, "lambda",  3, 5800.);
  cpl_table_set_double(table, "lambda",  4, 5900.);
  cpl_table_set_double(table, "lambda",  5, 6000.);
  cpl_table_set_double(table, "lambda",  6, 6100.);
  cpl_table_set_double(table, "lambda",  7, 6200.);
  cpl_table_set_double(table, "lambda",  8, 6300.);
  cpl_table_set_double(table, "lambda",  9, 6400.);
  cpl_table_set_double(table, "lambda", 10, 6500.);
  cpl_table_set_double(table, "lambda", 11, 6600.);
  cpl_table_set_double(table, "lambda", 12, 6700.);
  cpl_table_set_double(table, "lambda", 13, 6800.);
  cpl_table_set_double(table, "lambda", 14, 6900.);
  cpl_table_set_double(table, "lambda", 15, 7000.);
  cpl_table_set_double(table, "lambda", 16, 7100.);
  cpl_table_set_double(table, "lambda", 17, 7200.);
  cpl_table_set_double(table, "lambda", 18, 7300.);
  cpl_table_set_double(table, "lambda", 19, 7400.);
  cpl_table_set_double(table, "lambda", 20, 7500.);
  cpl_table_set_double(table, "lambda", 21, 8000.);
  cpl_table_set_double(table, "lambda", 22, 8500.);
  cpl_table_set_double(table, "lambda", 23, 9000.);
  cpl_table_set_double(table, "throughput",  0, 0.00E+00);
  cpl_table_set_double(table, "throughput",  1, 2.30E-01);
  cpl_table_set_double(table, "throughput",  2, 7.40E-01);
  cpl_table_set_double(table, "throughput",  3, 9.10E-01);
  cpl_table_set_double(table, "throughput",  4, 9.80E-01);
  cpl_table_set_double(table, "throughput",  5, 1.00E+00);
  cpl_table_set_double(table, "throughput",  6, 9.80E-01);
  cpl_table_set_double(table, "throughput",  7, 9.60E-01);
  cpl_table_set_double(table, "throughput",  8, 9.30E-01);
  cpl_table_set_double(table, "throughput",  9, 9.00E-01);
  cpl_table_set_double(table, "throughput", 10, 8.60E-01);
  cpl_table_set_double(table, "throughput", 11, 8.10E-01);
  cpl_table_set_double(table, "throughput", 12, 7.80E-01);
  cpl_table_set_double(table, "throughput", 13, 7.20E-01);
  cpl_table_set_double(table, "throughput", 14, 6.70E-01);
  cpl_table_set_double(table, "throughput", 15, 6.10E-01);
  cpl_table_set_double(table, "throughput", 16, 5.60E-01);
  cpl_table_set_double(table, "throughput", 17, 5.10E-01);
  cpl_table_set_double(table, "throughput", 18, 4.60E-01);
  cpl_table_set_double(table, "throughput", 19, 4.00E-01);
  cpl_table_set_double(table, "throughput", 20, 3.50E-01);
  cpl_table_set_double(table, "throughput", 21, 1.40E-01);
  cpl_table_set_double(table, "throughput", 22, 3.00E-02);
  cpl_table_set_double(table, "throughput", 23, 0.00E+00);

  muse_table *mt = muse_table_new();
  mt->table = table;
  mt->header = cpl_propertylist_new();
  cpl_propertylist_append_string(mt->header, "EXTNAME", "Cousins_R");
  return mt;
} /* muse_test_flux_cousins_r() */

void
muse_test_flux_do_calibrate(muse_pixtable *aPt, muse_table *aResponse,
                            cpl_table *aExtinction, muse_table *aTelluric)
{
  cpl_msg_debug(__func__, "Starting muse_flux_calibrate(%s extinction, "
                "%s telluric)...", aExtinction ? "with" : "without",
                aTelluric ? "with" : "without");
  double cputime1 = cpl_test_get_cputime(),
         time1 = cpl_test_get_walltime();
  cpl_error_code rc = muse_flux_calibrate(aPt, aResponse, aExtinction,
                                          aTelluric);
  cpl_test(rc == CPL_ERROR_NONE);
  double cputime2 = cpl_test_get_cputime(),
         time2 = cpl_test_get_walltime();
  cpl_msg_debug(__func__, "...done muse_flux_calibrate(), "
                "took %gs (CPU) %gs (wall-clock)", cputime2 - cputime1,
                time2 - time1);
  /* test success case of muse_pixtable_is_fluxcal() */
  cpl_errorstate ps = cpl_errorstate_get();
  cpl_boolean fluxcal = muse_pixtable_is_fluxcal(aPt);
  cpl_test(fluxcal);
  cpl_test(cpl_errorstate_is_equal(ps));
} /* muse_test_flux_do_calibrate() */

void
muse_test_flux_check_cubes(muse_datacube **aCubes, unsigned short aNCubes)
{
  /* construct image for statistics from the inner pixel, *
   * i.e. [2,2,*] (the central spectrum!), of the cube    */
  int nlambda = cpl_imagelist_get_size(aCubes[0]->data);
  cpl_image *image = cpl_image_new(nlambda, 4, CPL_TYPE_DOUBLE);
  double cdelt = muse_pfits_get_cd(aCubes[0]->header, 3, 3),
         crval = muse_pfits_get_crval(aCubes[0]->header, 3);
  const int windows[][2] = {
    {    1, nlambda }, /* full spectrum */
    {  650, 1050 }, /* blue line-free region, ~5610...6110 Angstrom */
    { 1651, 1735 }, /* telluric B-band */
    { 2075, 2222 } /* red line-free region, ~7395...7575 Angstrom */
  };
  cpl_stats *s[aNCubes * 4]; /* three methods, four windows */
  int i, /* method number / image row index */
      idxstat = 0; /* stats index */
  for (i = 1; i <= aNCubes; i++) {
    int idxcube = i - 1,
        imagerow = i + 1,
        l, /* wavelength (plane) index */
        err;
    for (l = 0; l < nlambda; l++) {
      if (i == 1) { /* fill lambda row */
        cpl_image_set(image, l + 1, 1, l * cdelt + crval);
      }
      double value = cpl_image_get(cpl_imagelist_get(aCubes[idxcube]->data, l),
                                   2, 2, &err);
      cpl_image_set(image, l + 1, imagerow, value);
    } /* for l (wavelengths) */
    muse_datacube_delete(aCubes[idxcube]);

    int k;
    for (k = 0; k < 4; k++) { /* statistics windows */
      cpl_stats *stats = cpl_stats_new_from_image_window(image, CPL_STATS_ALL,
                                                         windows[k][0], imagerow,
                                                         windows[k][1], imagerow);
      cpl_msg_debug(__func__, "Stats, method %d, window %d (%d...%d, "
                    "%.2f...%.2f): %.3f (%.3f) +/- %.3f, %.3f ... %.3f", i, k,
                    windows[k][0], windows[k][1],
                    cpl_image_get(image, windows[k][0], 1, &err),
                    cpl_image_get(image, windows[k][1], 1, &err),
                    cpl_stats_get_mean(stats), cpl_stats_get_median(stats),
                    cpl_stats_get_stdev(stats),
                    cpl_stats_get_min(stats), cpl_stats_get_max(stats));
      s[idxstat++] = stats;
    } /* for k */
  } /* for i (all aNCubes) */
#if 0
  cpl_plot_image_row(NULL, "w l", NULL, image, 1, 4, 1);
#endif
  cpl_image_delete(image);

  /* now do the testing of the various statistics: *
   * idx  cube  window                             *
   *   0  noext full                               *
   *   1  noext blue                               *
   *   2  noext B-band                             *
   *   3  noext red                                *
   *   4  ext   full                               *
   *   5  ext   blue                               *
   *   6  ext   B-band                             *
   *   7  ext   red                                *
   *   8  tell  full                               *
   *   9  tell  blue                               *
   *  10  tell  B-band                             *
   *  11  tell  red                                */
  /* flux increase with more corrections */
  cpl_test(cpl_stats_get_mean(s[0]) < cpl_stats_get_mean(s[4]));
  cpl_test(cpl_stats_get_mean(s[4]) < cpl_stats_get_mean(s[8]));
  /* flux increases more in the blue than in the red */
  double diffbluered_noext = cpl_stats_get_mean(s[1]) - cpl_stats_get_mean(s[3]),
         diffbluered_ext = cpl_stats_get_mean(s[5]) - cpl_stats_get_mean(s[7]);
  cpl_msg_debug(__func__, "diffbluered: %e (noext) %e (ext)",
                diffbluered_noext, diffbluered_ext);
  cpl_test(diffbluered_noext < diffbluered_ext &&
           diffbluered_ext / diffbluered_noext > 1.79);
  /* no flux change in blue or red regions between ext and tell */
  cpl_test(fabs(cpl_stats_get_mean(s[5]) - cpl_stats_get_mean(s[9])) < FLT_EPSILON);
  cpl_test(fabs(cpl_stats_get_mean(s[7]) - cpl_stats_get_mean(s[11])) < FLT_EPSILON);
  cpl_test(fabs(cpl_stats_get_stdev(s[5]) - cpl_stats_get_stdev(s[9])) < FLT_EPSILON);
  cpl_test(fabs(cpl_stats_get_stdev(s[7]) - cpl_stats_get_stdev(s[11])) < FLT_EPSILON);
  /* (strong) changes within the B-band */
  cpl_test(cpl_stats_get_mean(s[10]) / cpl_stats_get_mean(s[6]) > 1.06);
  cpl_test(cpl_stats_get_min(s[10]) / cpl_stats_get_min(s[6]) > 1.45);
  /* not so strong for the max value */
  cpl_test(cpl_stats_get_max(s[10]) / cpl_stats_get_max(s[6]) > 1.00 &&
           cpl_stats_get_max(s[10]) / cpl_stats_get_max(s[6]) < 1.06);
  /* check against reality: the GD71 reference spectrum has mean values of *
   * full   1.6275186e-15                                                  *
   * blue   1.9847039e-15            ==> blue / red = 1.553501088474       *
   * B-band 1.4732238e-15                blue / B-band = 1.3471842499422   *
   * red    1.2775684e-15                blue / full = 1.2194661861315     */
  double divbluered = cpl_stats_get_mean(s[9]) / cpl_stats_get_mean(s[11]),
         divblueB = cpl_stats_get_mean(s[9]) / cpl_stats_get_mean(s[10]),
         divbluefull = cpl_stats_get_mean(s[9]) / cpl_stats_get_mean(s[8]);
  cpl_msg_debug(__func__, "divs (tell): blue/red = %e, blue/B-band = %e, "
                "blue/full = %e", divbluered, divblueB, divbluefull);
  /* XXX why are they so low (0.8106, 0.8409, 0.9518), *
   *     the slope of the spectrum is too low!         */
  cpl_test(divbluered / 1.553501088474 < 1.05 && divbluered / 1.553501088474 > 0.81);
  cpl_test(divblueB / 1.3471842499422 < 1.05 && divblueB / 1.3471842499422 > 0.84);
  cpl_test(divbluefull / 1.2194661861315 < 1.05 && divbluefull / 1.2194661861315 > 0.95);
  for (i = 0; i < idxstat; i++) {
    cpl_stats_delete(s[i]);
  }
} /* muse_test_flux_check_cubes() */

#define PRINT_USAGE(rc)                                                        \
  fprintf(stderr, "\nUsage: %s [ -profile <gaussian | moffat | smoffat | "     \
          "circle | square | auto> ] [ -select <flux | distance> ] [ -smooth " \
          "<none | median | ppoly> ] DATACUBE_STD STD_FLUX_TABLE "             \
          "EXTINCTION_TABLE\n  The output STD_RESPONSE is written to a file "  \
          "starting with the same basename as DATACUBE_STD.\n\n\n", argv[0]);  \
  return (rc);

int
muse_test_flux_integrate_from_cube(int argc, char **argv)
{
  cpl_errorstate state = cpl_errorstate_get();

  cpl_msg_set_level(CPL_MSG_DEBUG); /* debug output is nice for this use case */

  /* defaults */
  char *incube = NULL,
       *inref = NULL,
       *inext = NULL;
  muse_flux_profile_type profile = MUSE_FLUX_PROFILE_AUTO;
  muse_flux_selection_type select = MUSE_FLUX_SELECT_NEAREST;
  muse_spectrum_smooth_type smooth = MUSE_SPECTRUM_SMOOTH_PPOLY;

  /* argument processing */
  int i;
  for (i = 1; i < argc; i++) {
    if (strncmp(argv[i], "-profile", 9) == 0) {
      /* skip to next arg to first slice number */
      i++;
      if (i < argc) {
        if (!strncmp(argv[i], "moffat", 7)) {
          profile = MUSE_FLUX_PROFILE_MOFFAT;
        } else if (!strncmp(argv[i], "smoffat", 8)) {
          profile = MUSE_FLUX_PROFILE_SMOFFAT;
        } else if (!strncmp(argv[i], "gaussian", 9)) {
          profile = MUSE_FLUX_PROFILE_GAUSSIAN;
        } else if (!strncmp(argv[i], "circle", 7)) {
          profile = MUSE_FLUX_PROFILE_CIRCLE;
        } else if (!strncmp(argv[i], "square", 7)) {
          profile = MUSE_FLUX_PROFILE_EQUAL_SQUARE;
        } else if (!strncmp(argv[i], "auto", 5)) {
          profile = MUSE_FLUX_PROFILE_AUTO;
        }
      } else {
        PRINT_USAGE(2);
      }
    } else if (strncmp(argv[i], "-select", 8) == 0) {
      /* skip to next arg to last slice number */
      i++;
      if (i < argc) {
        if (strncmp(argv[i], "flux", 5) == 0) {
          select = MUSE_FLUX_SELECT_BRIGHTEST;
        } else if (strncmp(argv[i], "distance", 9) == 0) {
          select = MUSE_FLUX_SELECT_NEAREST;
        }
      } else {
        PRINT_USAGE(3);
      }
    } else if (strncmp(argv[i], "-smooth", 8) == 0) {
      /* skip to next arg to last slice number */
      i++;
      if (i < argc) {
        if (strncmp(argv[i], "none", 5) == 0) {
          smooth = MUSE_SPECTRUM_SMOOTH_NONE;
        } else if (strncmp(argv[i], "median", 7) == 0) {
          smooth = MUSE_SPECTRUM_SMOOTH_MEDIAN;
        } else if (strncmp(argv[i], "ppoly", 6) == 0) {
          smooth = MUSE_SPECTRUM_SMOOTH_PPOLY;
        }
      } else {
        PRINT_USAGE(4);
      }
    } else if (strncmp(argv[i], "-", 1) == 0) { /* unallowed options */
      PRINT_USAGE(9);
    } else {
      if (incube && inref && inext) {
        break; /* we have the possible names, skip the rest */
      }
      if (!incube) {
        incube = argv[i]; /* set the name for the input cube */
      } else if (!inref) {
        inref = argv[i]; /* set the name for the input reference table */
      } else {
        inext = argv[i]; /* set the name for the input extinction table */
      }
    }
  } /* for i (all arguments) */

  muse_datacube *cube = muse_datacube_load(incube);
  if (!cube) {
    PRINT_USAGE(10);
  }
  /* copy stuff from muse_flux_integrate_std() */
  int nplane = cpl_imagelist_get_size(cube->data) / 2; /* central plane */
  cpl_image *cim = cpl_imagelist_get(cube->data, nplane);
  /* make sure to reject bad pixels before using object detection      *
   * using image statistics; different from muse_flux_integrate_std(), *
   * here we need to use NANs for that, since the saved-and-reloaded   *
   * cubes do not have a DQ component                                  */
  cpl_image_reject_value(cim, CPL_VALUE_NAN); /* mark NANs */
  double dsigmas[] = { 50., 30., 10., 5. };
  cpl_vector *vsigmas = cpl_vector_wrap(sizeof(dsigmas) / sizeof(double),
                                        dsigmas);
  cpl_size isigma = -1;
  cpl_apertures *apertures = cpl_apertures_extract(cim, vsigmas, &isigma);
  int napertures = apertures ? cpl_apertures_get_size(apertures) : 0;
  if (napertures < 1) {
    double med_dist,
           median = cpl_image_get_median_dev(cim, &med_dist),
           threshold = median + 5. * med_dist;
    cpl_msg_debug(__func__, "failed: %f +/- %f --> %f on plane %d", median,
                  med_dist, threshold, nplane + 1);
    cpl_vector_unwrap(vsigmas);
    cpl_apertures_delete(apertures);
    muse_datacube_delete(cube);
    PRINT_USAGE(11);
  }
  cpl_msg_debug(__func__, "The %.1f sigma threshold was used to find %d source%s"
                " on plane %d", cpl_vector_get(vsigmas, isigma), napertures,
                napertures == 1 ? "" : "s", nplane + 1);
  cpl_vector_unwrap(vsigmas);
  /* now do the flux integration and save the resulting file */
  muse_image *intimage = muse_flux_integrate_cube(cube, apertures, profile);
  cpl_apertures_delete(apertures);
  char *dot = strrchr(incube, '.');
  *dot = '\0'; /* replace the dot */
  char *fn = cpl_sprintf("%s_STD_FLUXES.fits", incube);
  muse_image_save(intimage, fn);
  cpl_msg_info(__func__, "Wrote intimage as \"%s\".", fn);
  cpl_free(fn);

  /* load the tables */
  cpl_frame *refframe = cpl_frame_new();
  cpl_frame_set_filename(refframe, inref);
  cpl_table *reftable = muse_postproc_load_nearest(cube->header, refframe,
                                                   20., 60., NULL, NULL),
            *exttable = cpl_table_load(inext, 1, 1);
  cpl_frame_delete(refframe);
  if (!reftable || !exttable) {
    muse_datacube_delete(cube);
    muse_image_delete(intimage);
    PRINT_USAGE(12);
  }
  muse_flux_reference_table_check(reftable);
  double airmass = muse_astro_airmass(cube->header);
  muse_flux_object *fo = muse_flux_object_new();
  fo->cube = cube;
  fo->intimage = intimage;
  fo->raref = muse_pfits_get_ra(cube->header);
  fo->decref = muse_pfits_get_dec(cube->header);
  muse_flux_response_compute(fo, select, airmass, reftable,
                             NULL, exttable);
  muse_flux_get_response_table(fo, smooth);
  muse_flux_compute_qc(fo);
  fn = cpl_sprintf("%s_STD_RESPONSE.fits", incube);
  muse_table_save(fo->response, fn);
  cpl_msg_info(__func__, "Wrote response as \"%s\".", fn);
  cpl_free(fn);
  muse_flux_get_telluric_table(fo);
  fn = cpl_sprintf("%s_STD_TELLURIC.fits", incube);
  muse_table_save(fo->telluric, fn);
  cpl_msg_info(__func__, "Wrote telluric as \"%s\".", fn);
  cpl_free(fn);
  muse_flux_object_delete(fo);
  cpl_table_delete(reftable);
  cpl_table_delete(exttable);

  return cpl_errorstate_is_equal(state) ? CPL_ERROR_NONE : cpl_error_get_code();
} /* muse_test_flux_integrate_from_cube() */

/*----------------------------------------------------------------------------*/
/**
  @brief    test program to check that the functions from the muse_flux module
            do what they should when called with the necessary data

  This program explicitely tests
    muse_flux_object_new
    muse_flux_object_delete
    muse_flux_reference_table_check
    muse_flux_response_interpolate
    muse_flux_integrate_cube
    muse_flux_integrate_std
    muse_flux_response_compute
    muse_flux_get_response_table
    muse_flux_get_telluric_table
    muse_flux_compute_qc
    muse_flux_compute_qc_zp
    muse_flux_calibrate
 */
/*----------------------------------------------------------------------------*/
int main(int argc, char **argv)
{
  cpl_error_code rc = CPL_ERROR_NONE;
  if (argc > 1) {
    /* If there are arguments, misuse this test program as flux integrator. */
    cpl_init(CPL_INIT_DEFAULT);
    rc = muse_test_flux_integrate_from_cube(argc, argv);
    cpl_end();
    return rc;
  }

  /* start as test program */
  cpl_test_init(PACKAGE_BUGREPORT, CPL_MSG_DEBUG);
  cpl_table *ref = cpl_table_load(BASEFILENAME"_reference_gd71sm.fits", 1, 1),
            *extinction = cpl_table_load(BASEFILENAME"_extinction.fits", 1, 1);

  /* test muse_flux_reference_table_check() */
  cpl_test(muse_flux_reference_table_check(ref) == CPL_ERROR_NONE);
  cpl_errorstate state = cpl_errorstate_get();
  cpl_test(muse_flux_reference_table_check(NULL) == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  /* test with HST table */
  cpl_table *refhst = cpl_table_load(BASEFILENAME"_gd153_stisnic_004.fits", 1, 1),
            *refhst2 = cpl_table_duplicate(refhst);
  cpl_test(muse_flux_reference_table_check(refhst) == CPL_ERROR_NONE);
  cpl_test(cpl_table_has_column(refhst, "lambda"));
  cpl_test(cpl_table_has_column(refhst, "flux"));
  cpl_table_delete(refhst);
  /* test again with broken HST table */
  cpl_table_name_column(refhst2, "WAVELENGTH", "wavel");
  state = cpl_errorstate_get();
  cpl_test(muse_flux_reference_table_check(refhst2) == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  cpl_table_name_column(refhst2, "wavel", "WAVELENGTH");
  cpl_table_set_column_unit(refhst2, "FLUX", "FLAMM");
  cpl_test(muse_flux_reference_table_check(refhst2) == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  cpl_table_set_column_unit(refhst2, "WAVELENGTH", NULL);
  cpl_table_set_column_unit(refhst2, "FLUX", NULL);
  cpl_test(muse_flux_reference_table_check(refhst2) == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  cpl_table_delete(refhst2);
  /* test with alternative unit string */
  cpl_table *ref2 = cpl_table_duplicate(ref);
  cpl_table_set_column_unit(ref2, "flux", "erg/s/cm^2/Angstrom");
  cpl_test(muse_flux_reference_table_check(ref2) == CPL_ERROR_NONE);
  /* test without unit strings */
  cpl_table_set_column_unit(ref2, "lambda", NULL);
  cpl_table_set_column_unit(ref2, "flux", NULL);
  state = cpl_errorstate_get();
  cpl_test(muse_flux_reference_table_check(ref2) == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  /* test with (made-up) fluxerr column, but no unit */
  cpl_table_set_column_unit(ref2, "lambda", "Angstrom"); /* reset correct units */
  cpl_table_set_column_unit(ref2, "flux", "erg/s/cm**2/Angstrom");
  cpl_table_new_column(ref2, "fluxerr", CPL_TYPE_DOUBLE);
  cpl_test(muse_flux_reference_table_check(ref2) == CPL_ERROR_NONE);
  cpl_test(!cpl_table_has_column(ref2, "fluxerr")); /* get erased */
  cpl_table_new_column(ref2, "fluxerr", CPL_TYPE_DOUBLE);
  cpl_table_fill_column_window(ref2, "fluxerr", 0, cpl_table_get_nrow(ref2), 1e-3);
  cpl_table_set_column_unit(ref2, "fluxerr", "erg/s/cm**2/Angstrom");
  cpl_test(muse_flux_reference_table_check(ref2) == CPL_ERROR_NONE);
  cpl_test(cpl_table_has_column(ref2, "fluxerr")); /* now still there */
  /* now test with table where all columns are floats */
  cpl_table *ref3 = cpl_table_duplicate(ref2);
  cpl_table_name_column(ref3, "lambda", "lambda_d");
  cpl_table_name_column(ref3, "flux", "flux_d");
  cpl_table_name_column(ref3, "fluxerr", "fluxerr_d");
  cpl_table_cast_column(ref3, "lambda_d", "lambda", CPL_TYPE_FLOAT);
  cpl_table_cast_column(ref3, "flux_d", "flux", CPL_TYPE_FLOAT);
  cpl_table_cast_column(ref3, "fluxerr_d", "fluxerr", CPL_TYPE_FLOAT);
  cpl_table_erase_column(ref3, "lambda_d");
  cpl_table_erase_column(ref3, "flux_d");
  cpl_table_erase_column(ref3, "fluxerr_d");
  cpl_test(muse_flux_reference_table_check(ref3) == CPL_ERROR_NONE);
  cpl_test_zero(cpl_table_compare_structure(ref2, ref3)); /* now all doubles */
  cpl_table_delete(ref2);
  cpl_table_delete(ref3);

  state = cpl_errorstate_get();
  muse_pixtable *pt = muse_pixtable_load(BASEFILENAME"_pixtable_gd71sm.fits");
  cpl_test_nonnull(pt);
  cpl_errorstate_set(state); /* hide error from missing DQ and Origin columns */
  cpl_propertylist_append_string(pt->header, "ESO OBS TARG NAME", "GD71"); /* for QC */
  muse_pixtable *ptdupe = muse_pixtable_duplicate(pt);
  /* add (clean) DQ and (empty) origin columns back *
   * which were removed to save space               */
  cpl_table_new_column(pt->table, MUSE_PIXTABLE_DQ, CPL_TYPE_INT);
  cpl_table_fill_column_window_int(pt->table, MUSE_PIXTABLE_DQ,
                                   0, muse_pixtable_get_nrow(pt), 0);
  cpl_table_new_column(pt->table, MUSE_PIXTABLE_ORIGIN, CPL_TYPE_INT);
  cpl_table_fill_column_window_int(pt->table, MUSE_PIXTABLE_ORIGIN,
                                   0, muse_pixtable_get_nrow(pt), 0);
  double airmass = muse_astro_airmass(pt->header);

  muse_flux_object *fluxobj = muse_flux_object_new();
  cpl_test_nonnull(fluxobj);
  /* flux integration using Moffat profile fit */
  state = cpl_errorstate_get();
  rc = muse_flux_integrate_std(pt, MUSE_FLUX_PROFILE_MOFFAT, fluxobj);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(fluxobj->cube);
  cpl_test_nonnull(fluxobj->intimage);
  cpl_test(cpl_imagelist_get_size(fluxobj->cube->data)
           == cpl_image_get_size_x(fluxobj->intimage->data));
  /* compute the total flux recovered over the wavelength range ~4800 to 9300 */
  cpl_stats_mode smode = CPL_STATS_FLUX | CPL_STATS_MEDIAN;
  cpl_stats *s = cpl_stats_new_from_image_window(fluxobj->intimage->data, smode,
                                                 201, 1, 3801, 1);
  double ftot = cpl_stats_get_flux(s),
         fmedian = cpl_stats_get_median(s);
  cpl_stats_delete(s);
  /* extract the source on the same sigma was done above     *
   * (the cube is 3961 pixels long, so use plane index 1980) */
  cpl_image *plane = cpl_imagelist_get(fluxobj->cube->data, 1980);
  cpl_apertures *ap = cpl_apertures_extract_sigma(plane, 50.);
  /* Before starting to mess with the pixel table, try integrating the    *
   * flux using the other profile types, too. Since the above tested      *
   * muse_flux_integrate_std(), we can now use muse_flux_integrate_cube() *
   * on the existing cube which is much faster.                           */
  int i;
  for (i = MUSE_FLUX_PROFILE_AUTO; i >= MUSE_FLUX_PROFILE_MOFFAT; i--) {
    if (i == MUSE_FLUX_PROFILE_MOFFAT) {
      /* in the last iteration, for the moffat profile, test the (success) *
       * case where the DIMM measurements are missing from the cube header */
      cpl_propertylist_erase_regexp(fluxobj->cube->header, "ESO TEL AMBI FWHM ", 0);
    }
    state = cpl_errorstate_get();
    muse_image *intimage = muse_flux_integrate_cube(fluxobj->cube, ap, i);
    cpl_test(cpl_errorstate_is_equal(state));
    cpl_test_nonnull(intimage);
    cpl_test(cpl_imagelist_get_size(fluxobj->cube->data)
             == cpl_image_get_size_x(intimage->data));
    s = cpl_stats_new_from_image_window(intimage->data, smode, 201, 1, 3801, 1);
    /* should be at least comparable to Moffat integration */
    cpl_msg_debug(__func__, "flux %g / %g\n\tmedian %g / %g",
                  ftot, cpl_stats_get_flux(s), fmedian, cpl_stats_get_median(s));
    /* the Gaussian integrates less flux than the other methods... */
    double limit1 = i == MUSE_FLUX_PROFILE_GAUSSIAN ? 0.18 : 0.04,
           limit2 = i == MUSE_FLUX_PROFILE_GAUSSIAN ? 0.15 : 0.02;
    cpl_test_rel(ftot, cpl_stats_get_flux(s), limit1);
    cpl_test_rel(fmedian, cpl_stats_get_median(s), limit2);
    cpl_stats_delete(s);
    muse_image_delete(intimage);

    /* for one of the calls, test the case of a NAN-converted cube */
    if (i == MUSE_FLUX_PROFILE_GAUSSIAN) {
      /* duplicate the cube */
      muse_datacube *cube2 = cpl_calloc(1, sizeof(muse_datacube));
      cube2->header = cpl_propertylist_duplicate(fluxobj->cube->header);
      cube2->data = cpl_imagelist_duplicate(fluxobj->cube->data);
      cube2->dq = cpl_imagelist_duplicate(fluxobj->cube->dq);
      cube2->stat = cpl_imagelist_duplicate(fluxobj->cube->stat);
      muse_datacube_convert_dq(cube2);

      state = cpl_errorstate_get();
      intimage = muse_flux_integrate_cube(cube2, ap, i);
      cpl_test(cpl_errorstate_is_equal(state));
      cpl_test_nonnull(intimage);
      cpl_test(cpl_imagelist_get_size(fluxobj->cube->data)
               == cpl_image_get_size_x(intimage->data));
      s = cpl_stats_new_from_image_window(intimage->data, smode, 201, 1, 3801, 1);
      /* should be at least comparable to Moffat integration */
      cpl_msg_debug(__func__, "flux %g / %g\n\tmedian %g / %g",
                    ftot, cpl_stats_get_flux(s), fmedian, cpl_stats_get_median(s));
      /* the Gaussian integrates less flux than the other methods... */
      limit1 = 0.18,
      limit2 = 0.15;
      cpl_test_rel(ftot, cpl_stats_get_flux(s), limit1);
      cpl_test_rel(fmedian, cpl_stats_get_median(s), limit2);
      cpl_stats_delete(s);
      muse_image_delete(intimage);
      muse_datacube_delete(cube2);
    } /* if MUSE_FLUX_PROFILE_GAUSSIAN */
  } /* for i (all other integration profiles) */

  /* test with pixel table with the other instrument modes */
  const char *mode[] = { "WFM-NOAO-E", "WFM-AO-N", "WFM-AO-E", "WFM-AO-N",
                         "NFM-AO-N", NULL };
  for (i = 0; mode[i] != NULL; i++) {
    muse_pixtable *pt3 = muse_pixtable_duplicate(pt);
    /* set mode */
    cpl_propertylist_update_string(pt3->header, "ESO INS MODE", mode[i]);
    cpl_msg_debug(__func__, "original mode: %s, updated mode: %s",
                  muse_pfits_get_insmode(pt->header),
                  muse_pfits_get_insmode(pt3->header));
    muse_flux_object *fobj3 = muse_flux_object_new();
    state = cpl_errorstate_get();
    rc = muse_flux_integrate_std(pt3, MUSE_FLUX_PROFILE_MOFFAT, fobj3);
    cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
    cpl_test_nonnull(fobj3->cube);
#if 0
    char *fn3 = cpl_sprintf("cube_%s_3.fits", mode[i]);
    muse_datacube_save(fobj3->cube, fn3);
    cpl_free(fn3);
#endif
    cpl_test_nonnull(fobj3->intimage);
    cpl_test(cpl_imagelist_get_size(fobj3->cube->data)
             == cpl_image_get_size_x(fobj3->intimage->data));
    if (!strstr(mode[i], "NFM")) {
      /* should give identical integration image as original, if not NFM */
      cpl_test_image_rel(fobj3->intimage->data, fluxobj->intimage->data,
                         FLT_EPSILON);
      cpl_test_image_rel(fobj3->intimage->stat, fluxobj->intimage->stat,
                         FLT_EPSILON);
    }
    /* done for this mode, if this is data pretending to be non-AO */
    if (strstr(mode[i], "NOAO")) {
      muse_flux_object_delete(fobj3);
      muse_pixtable_delete(pt3);
      continue;
    }

    /* for the AO modes, also test with pixel table containing EURO3D_NOTCH_NAD */
    muse_basicproc_mask_notch_filter(pt3, 3); /* hack! */
    cpl_table_select_all(pt3->table);
    muse_flux_object *fobj4 = muse_flux_object_new();
    state = cpl_errorstate_get();
    rc = muse_flux_integrate_std(pt3, MUSE_FLUX_PROFILE_MOFFAT, fobj4);
    cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
    cpl_test_nonnull(fobj4->cube);
#if 0
    char *fn4 = cpl_sprintf("cube_%s_4.fits", mode[i]);
    muse_datacube_save(fobj4->cube, fn4);
    cpl_free(fn4);
#endif
    cpl_test_nonnull(fobj4->intimage);
    cpl_test(cpl_imagelist_get_size(fobj4->cube->data)
             == cpl_image_get_size_x(fobj4->intimage->data));
    if (!strstr(mode[i], "NFM")) {
      cpl_test_image_rel(fobj4->intimage->data, fobj3->intimage->data,
                         FLT_EPSILON);
      cpl_test_image_rel(fobj4->intimage->stat, fobj3->intimage->stat,
                         FLT_EPSILON);
    }
    muse_flux_object_delete(fobj4);
    muse_flux_object_delete(fobj3);
    muse_pixtable_delete(pt3);
  } /* for i (other instrument modes) */

  /* test failure cases of muse_flux_integrate_std(), too */
  muse_flux_object *fobj3 = muse_flux_object_new();
  state = cpl_errorstate_get();
  rc = muse_flux_integrate_std(pt, MUSE_FLUX_PROFILE_AUTO + 1, fobj3);
  cpl_test(rc == CPL_ERROR_ILLEGAL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  rc = muse_flux_integrate_std(NULL, MUSE_FLUX_PROFILE_MOFFAT, fobj3);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  muse_flux_object_delete(fobj3);
  rc = muse_flux_integrate_std(pt, MUSE_FLUX_PROFILE_MOFFAT, NULL);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* test with pixel table with constant values, to trigger detection failure */
  muse_pixtable *ptc = muse_pixtable_duplicate(pt);
  muse_pixtable_restrict_wavelength(ptc, 6950., 7050.);
  cpl_table_fill_column_window_float(ptc->table, MUSE_PIXTABLE_DATA,
                                     0, cpl_table_get_nrow(ptc->table), 100);
  fobj3 = muse_flux_object_new();
  state = cpl_errorstate_get();
  rc = muse_flux_integrate_std(ptc, MUSE_FLUX_PROFILE_MOFFAT, fobj3);
  cpl_test(rc == CPL_ERROR_DATA_NOT_FOUND && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  muse_flux_object_delete(fobj3);
  muse_pixtable_delete(ptc);
  /* and failure cases for muse_flux_integrate_cube() */
  state = cpl_errorstate_get();
  muse_image *image = muse_flux_integrate_cube(NULL, ap, MUSE_FLUX_PROFILE_MOFFAT);
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(image);
  image = muse_flux_integrate_cube(fluxobj->cube, NULL, MUSE_FLUX_PROFILE_MOFFAT);
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(image);
  image = muse_flux_integrate_cube(fluxobj->cube, ap, MUSE_FLUX_PROFILE_AUTO + 1);
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() ==CPL_ERROR_ILLEGAL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(image);
  cpl_apertures_delete(ap);

  /* test success of muse_flux_response_compute() for reasonable data */
  state = cpl_errorstate_get();
  rc = muse_flux_response_compute(fluxobj, MUSE_FLUX_SELECT_BRIGHTEST, airmass,
                                  ref, NULL, extinction);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(fluxobj->reference);
  cpl_test_nonnull(fluxobj->sensitivity);

  /* create a filter table for Cousins_R for testing */
  muse_table *cousinsr = muse_test_flux_cousins_r();

  /* test muse_flux_compute_qc(), in this case just that it runs    *
   * through (and manually, that it prints the QC parameter values) */
  state = cpl_errorstate_get();
  rc = muse_flux_compute_qc(fluxobj);
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  /* test failures of muse_flux_compute_qc() */
  state = cpl_errorstate_get();
  rc = muse_flux_compute_qc(NULL);
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_table *stmp = fluxobj->sensitivity;
  fluxobj->sensitivity = NULL;
  rc = muse_flux_compute_qc(fluxobj); /* neither sensitivity nor response */
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "Cousins_R");
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  fluxobj->sensitivity = stmp;

  /* check muse_flux_get_response_table() */
  state = cpl_errorstate_get();
  rc = muse_flux_get_response_table(fluxobj, MUSE_SPECTRUM_SMOOTH_MEDIAN);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(fluxobj->response);
  cpl_test_nonnull(fluxobj->response->table);
  cpl_test_nonnull(fluxobj->response->header);
  cpl_test(cpl_table_has_column(fluxobj->response->table, "lambda") &&
           cpl_table_has_column(fluxobj->response->table, "response") &&
           cpl_table_has_column(fluxobj->response->table, "resperr"));
  cpl_test(cpl_table_get_nrow(fluxobj->response->table)
           == cpl_table_get_nrow(fluxobj->sensitivity));
  /* check resulting responses at three wavelengths, values were read *
   * off by eye from the sensitivity plots around these wavelengths   */
  double r1 = muse_flux_response_interpolate(fluxobj->response->table, 5000., NULL,
                                             MUSE_FLUX_RESP_FLUX),
         r2 = muse_flux_response_interpolate(fluxobj->response->table, 7000., NULL,
                                             MUSE_FLUX_RESP_FLUX),
         r3 = muse_flux_response_interpolate(fluxobj->response->table, 9000., NULL,
                                             MUSE_FLUX_RESP_FLUX);
  cpl_msg_debug(__func__, "responses computed (WFM-NOAO-N, median): %f...%f..."
                "%f", r1, r2, r3);
  cpl_test_abs(r1, 41.676, 0.02);
  cpl_test_abs(r2, 41.581, 0.02);
  cpl_test_abs(r3, 41.422, 0.02);
  /* test with pixel table with the all instrument modes; here, *
   * also use non-AO nominal mode again, since the above test   *
   * was without flat-field correction flat in the header       */
  const char *modeall[] = { "WFM-NOAO-N", "WFM-NOAO-E", "WFM-AO-N", "WFM-AO-E",
                            "WFM-AO-N", "NFM-AO-N", NULL };
  for (i = 0; modeall[i] != NULL; i++) {
    /* "duplicate" the flux object, but really only copy cube header *
     * and sensitivity; do that twice, one for each smoothing type   */
    muse_flux_object *fobj3p = muse_flux_object_new(),
                     *fobj4p = muse_flux_object_new();
    fobj3p->cube = cpl_calloc(1, sizeof(muse_datacube));
    fobj3p->cube->header = cpl_propertylist_duplicate(fluxobj->cube->header);
    fobj3p->sensitivity = cpl_table_duplicate(fluxobj->sensitivity);
    fobj4p->cube = cpl_calloc(1, sizeof(muse_datacube));
    fobj4p->cube->header = cpl_propertylist_duplicate(fluxobj->cube->header);
    fobj4p->sensitivity = cpl_table_duplicate(fluxobj->sensitivity);
    /* set mode, muse_flux_get_response_table() needs it in the cube header */
    cpl_propertylist_update_string(fobj3p->cube->header, "ESO INS MODE",
                                   modeall[i]);
    cpl_propertylist_update_string(fobj4p->cube->header, "ESO INS MODE",
                                   modeall[i]);
    /* also set flat-field corrected flag */
    cpl_propertylist_update_bool(fobj3p->cube->header, MUSE_HDR_FLUX_FFCORR,
                                 CPL_TRUE);
    cpl_propertylist_update_bool(fobj4p->cube->header, MUSE_HDR_FLUX_FFCORR,
                                 CPL_TRUE);

    state = cpl_errorstate_get();
    rc = muse_flux_get_response_table(fobj3p, MUSE_SPECTRUM_SMOOTH_MEDIAN);
    cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
    cpl_test_nonnull(fobj3p->response);
    cpl_test_nonnull(fobj3p->response->table);
    cpl_test_nonnull(fobj3p->response->header);
    /* the response values should not have changed */
    double rr1 = muse_flux_response_interpolate(fobj3p->response->table, 5000.,
                                                NULL, MUSE_FLUX_RESP_FLUX),
           rr2 = muse_flux_response_interpolate(fobj3p->response->table, 7000.,
                                                NULL, MUSE_FLUX_RESP_FLUX),
           rr3 = muse_flux_response_interpolate(fobj3p->response->table, 9000.,
                                                NULL, MUSE_FLUX_RESP_FLUX);
    cpl_msg_debug(__func__, "responses computed (%s, median): %f...%f...%f",
                  muse_pfits_get_insmode(fobj3p->cube->header), rr1, rr2, rr3);
    cpl_test_abs(r1, rr1, FLT_EPSILON);
    cpl_test_abs(r2, rr2, FLT_EPSILON);
    cpl_test_abs(r3, rr3, FLT_EPSILON);
    cpl_test_abs(rr1, 41.676, 0.02);
    cpl_test_abs(rr2, 41.581, 0.02);
    cpl_test_abs(rr3, 41.422, 0.02);
    muse_flux_object_delete(fobj3p);

    state = cpl_errorstate_get();
    rc = muse_flux_get_response_table(fobj4p, MUSE_SPECTRUM_SMOOTH_PPOLY);
    cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
    cpl_test_nonnull(fobj4p->response);
    cpl_test_nonnull(fobj4p->response->table);
    cpl_test_nonnull(fobj4p->response->header);
    /* again, approximately the same output response */
    rr1 = muse_flux_response_interpolate(fobj4p->response->table, 5000., NULL,
                                         MUSE_FLUX_RESP_FLUX),
    rr2 = muse_flux_response_interpolate(fobj4p->response->table, 7000., NULL,
                                         MUSE_FLUX_RESP_FLUX),
    rr3 = muse_flux_response_interpolate(fobj4p->response->table, 9000., NULL,
                                         MUSE_FLUX_RESP_FLUX);
    cpl_msg_debug(__func__, "responses computed (%s, ppoly): %f...%f...%f",
                  muse_pfits_get_insmode(fobj4p->cube->header), rr1, rr2, rr3);
    cpl_test_abs(rr1, 41.676, 0.02);
    cpl_test_abs(rr2, 41.581, 0.02);
    cpl_test_abs(rr3, 41.422, 0.02);
    muse_flux_object_delete(fobj4p);
  } /* for i (other instrument modes) */

  /* test muse_flux_compute_qc() for the same dataset, now we can check *
   * that we get at least similar values in the output header as the    *
   * values from the unsmoothed version before;                         *
   * take this chance to test the case of (partially) invalid rows      */
  state = cpl_errorstate_get();
  cpl_table_set_invalid(fluxobj->response->table, "lambda", 10);
  cpl_table_set_invalid(fluxobj->response->table, "response", 12);
  rc = muse_flux_compute_qc(fluxobj);
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  const char *starname = cpl_propertylist_get_string(fluxobj->response->header,
                                                     "ESO QC STANDARD STARNAME");
  cpl_test_eq_string(starname, "GD71");
  cpl_propertylist_erase_regexp(fluxobj->response->header,
                                "ESO QC STANDARD THRU", 1);
  cpl_test_eq(cpl_propertylist_get_size(fluxobj->response->header), 4);
  float thru[] = { 0.3865, 0.2938, 0.2540, 0.1662, 0.1665 };
  for (i = 0; i < 5; i++) {
    if (i == 1) {
      continue; /* skip entry at 6000 Angstrom that's not written any more */
    }
    char *kw = cpl_sprintf("ESO QC STANDARD THRU%04.0f", (i + 5.) * 1000.);
    cpl_test_abs(cpl_propertylist_get_float(fluxobj->response->header, kw),
                 thru[i], 0.01);
    cpl_free(kw);
  } /* for i (all 5 wavelengths) */
  /* special case: OBS.TARG.NAME missing from cube header */
  char *obstarg = cpl_strdup(muse_pfits_get_targname(fluxobj->cube->header));
  cpl_propertylist_erase(fluxobj->cube->header, "ESO OBS TARG NAME");
  /* check that it works when deleting the new clumn again */
  cpl_table_erase_column(fluxobj->response->table, "throughput");
  state = cpl_errorstate_get();
  rc = muse_flux_compute_qc(fluxobj);
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  cpl_test_eq_string(cpl_propertylist_get_string(fluxobj->response->header,
                                                 QC_STD_NAME), "UNKNOWN");
  cpl_propertylist_append_string(fluxobj->cube->header, "ESO OBS TARG NAME",
                                 obstarg);
  cpl_free(obstarg);
  /* test zeropoint calculation */
  state = cpl_errorstate_get();
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "Cousins_R");
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  float zp = cpl_propertylist_get_float(fluxobj->response->header,
                                        "ESO QC STANDARD ZP R");
  /* the INM apparently simulated a nice throughput of 27.3% in the R-band */
  cpl_test(isfinite(zp));
#if CPL_VERSION_CODE < 459009 /* before than CPL v7.1.1b3 */
  cpl_test_rel(zp, 1.41028, 1e-5);
#else /* with the changes to cpl_fit_lvmq() the values of the Moffat   *
       * extraction seem to have changed enough to require a new value */
  cpl_test_rel(zp, 1.41059, 1e-5);
#endif
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, NULL);
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  float zp2 = cpl_propertylist_get_float(fluxobj->response->header,
                                         "ESO QC STANDARD ZP UNKNOWN");
  cpl_test_eq(zp, zp2);
  /* name should not matter */
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "SDSS_g");
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  zp2 = cpl_propertylist_get_float(fluxobj->response->header,
                                   "ESO QC STANDARD ZP g");
  cpl_test_eq(zp, zp2);
  /* name without underscore */
  cpl_propertylist_erase(fluxobj->response->header,
                         "ESO QC STANDARD ZP UNKNOWN");
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "CousinsR");
  cpl_test(cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NONE);
  zp2 = cpl_propertylist_get_float(fluxobj->response->header,
                                   "ESO QC STANDARD ZP UNKNOWN");
  cpl_test_eq(zp, zp2);
  /* failure cases of muse_flux_compute_qc_zp() */
  state = cpl_errorstate_get();
  rc = muse_flux_compute_qc_zp(NULL, cousinsr, "Cousins_R");
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_table_delete(fluxobj->reference);
  fluxobj->reference = NULL;
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "Cousins_R");
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  fluxobj->reference = cpl_table_duplicate(ref);
  /* test when both sensitivity and response curve are missing */
  cpl_table *senssave = fluxobj->sensitivity;
  muse_table *respsave = fluxobj->response;
  fluxobj->sensitivity = NULL;
  fluxobj->response = NULL;
  rc = muse_flux_compute_qc_zp(fluxobj, cousinsr, "Cousins_R");
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  fluxobj->sensitivity = senssave;
  fluxobj->response = respsave;
  rc = muse_flux_compute_qc_zp(fluxobj, NULL, "Cousins_R");
  cpl_test(!cpl_errorstate_is_equal(state) && rc == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  muse_table_delete(cousinsr);

  /* check muse_flux_get_telluric_table() */
  rc = muse_flux_get_telluric_table(fluxobj);
  cpl_test_nonnull(fluxobj->telluric);
  cpl_test_nonnull(fluxobj->telluric->table);
  cpl_test_nonnull(fluxobj->telluric->header);
  cpl_test(cpl_table_has_column(fluxobj->telluric->table, "lambda") &&
           cpl_table_has_column(fluxobj->telluric->table, "ftelluric") &&
           cpl_table_has_column(fluxobj->telluric->table, "ftellerr"));
  cpl_test(cpl_table_get_nrow(fluxobj->telluric->table)
           < cpl_table_get_nrow(fluxobj->sensitivity));
  double mean = cpl_table_get_column_mean(fluxobj->telluric->table, "ftelluric"),
         median = cpl_table_get_column_median(fluxobj->telluric->table, "ftelluric"),
         stdev = cpl_table_get_column_stdev(fluxobj->telluric->table, "ftelluric");
  cpl_msg_debug(__func__, "telluric correction statistics: %f (%f) +/- %f",
                mean, median, stdev);
  cpl_test(fabs(mean - 1.) < 0.092 && fabs(median - 1.) < 0.032 &&
           stdev < 0.16);

  /* try again, but selecting the nearest star, should give  *
   * exactly the same result, since we have only 1 star here */
  cpl_table *sens1 = fluxobj->sensitivity;
  cpl_table_delete(fluxobj->tellbands);
  fluxobj->sensitivity = NULL;
  fluxobj->tellbands = NULL;
  /* also clean up the reference table for the next try */
  cpl_table_delete(fluxobj->reference);
  fluxobj->reference = NULL;
  state = cpl_errorstate_get();
  fluxobj->raref = 15.8871528; /* RA of GD71 */
  fluxobj->decref = 88.1150583; /* DEC of GD71 */
  rc = muse_flux_response_compute(fluxobj, MUSE_FLUX_SELECT_NEAREST, airmass,
                                  ref, NULL, extinction);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(fluxobj->reference);
  cpl_test_nonnull(fluxobj->sensitivity);
  /* sens1 has the throughput column from QC, so delete it first */
  cpl_table_erase_column(sens1, "throughput");
  cpl_test_zero(cpl_table_compare_structure(sens1, fluxobj->sensitivity));
  /* compare both tables row by row */
  int nrows1 = cpl_table_get_nrow(sens1),
      nrows2 = cpl_table_get_nrow(fluxobj->sensitivity);
  cpl_test_eq(nrows1, nrows2);
  cpl_array *acol = cpl_table_get_column_names(sens1);
  for (i = 0; i < 4; i++) { /* check cols lambda, sens, serr, and dq only */
    const char *colname = cpl_array_get_string(acol, i);
    cpl_type t = cpl_table_get_column_type(sens1, colname);
    cpl_array *a1, *a2;
    if (t == CPL_TYPE_INT) {
      a1 = cpl_array_wrap_int(cpl_table_get_data_int(sens1, colname), nrows1);
      a2 = cpl_array_wrap_int(cpl_table_get_data_int(fluxobj->sensitivity, colname),
                              nrows2);
    } else {
      a1 = cpl_array_wrap_double(cpl_table_get_data_double(sens1, colname), nrows1);
      a2 = cpl_array_wrap_double(cpl_table_get_data_double(fluxobj->sensitivity,
                                                           colname), nrows2);
    }
    cpl_test_array_abs(a1, a2, DBL_EPSILON);
    cpl_array_unwrap(a1);
    cpl_array_unwrap(a2);
  } /* for i (all column names) */
  cpl_array_delete(acol);
  cpl_table_delete(sens1);

  /* duplicate the telluric regions from above,      *
   * they are the defaults, so nothing should change */
  cpl_table *tellreg = cpl_table_duplicate(fluxobj->tellbands);
  muse_flux_object *fo2 = muse_flux_object_new();
  fo2->cube = fluxobj->cube;
  fo2->intimage = fluxobj->intimage;
  state = cpl_errorstate_get();
  rc = muse_flux_response_compute(fo2, MUSE_FLUX_SELECT_BRIGHTEST, airmass, ref,
                                  tellreg, extinction);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_table_delete(tellreg);
  /* check that some entries in the sensitivity table are identical */
  int nrow2 = cpl_table_get_nrow(fo2->sensitivity);
  cpl_msg_debug(__func__, "test with tellbands gave %d rows", nrow2);
  cpl_test(cpl_table_get_nrow(fluxobj->sensitivity) == nrow2);
  int err, err2;
  cpl_test(cpl_table_get(fluxobj->sensitivity, "sens", nrow2/100, &err)
           == cpl_table_get(fo2->sensitivity, "sens", nrow2/100, &err2)
           && err == err2);
  cpl_test(cpl_table_get(fluxobj->sensitivity, "sens", nrow2/10, &err)
           == cpl_table_get(fo2->sensitivity, "sens", nrow2/10, &err2)
           && err == err2);
  cpl_test(cpl_table_get(fluxobj->sensitivity, "sens", nrow2/5, &err)
           == cpl_table_get(fo2->sensitivity, "sens", nrow2/5, &err2)
           && err == err2);
  cpl_test(cpl_table_get(fluxobj->sensitivity, "sens", nrow2 - 10, &err)
           == cpl_table_get(fo2->sensitivity, "sens", nrow2 - 10, &err2)
           && err == err2);
  cpl_test(cpl_table_get(fluxobj->sensitivity, "serr", nrow2/10, &err)
           == cpl_table_get(fo2->sensitivity, "serr", nrow2/10, &err2)
           && err == err2);
  fo2->cube = NULL; /* reset the pointers again to not delete them yet */
  fo2->intimage = NULL;
  muse_flux_object_delete(fo2);

  cpl_msg_info(__func__, "using pixel table with %"CPL_SIZE_FORMAT" rows, "
               "response table with %"CPL_SIZE_FORMAT" entries",
               muse_pixtable_get_nrow(pt),
               cpl_table_get_nrow(fluxobj->response->table));

  /* duplicate the relevant columns to be able to compare before/after */
  cpl_table_duplicate_column(pt->table, "data_orig",
                             pt->table, MUSE_PIXTABLE_DATA);
  cpl_table_duplicate_column(pt->table, "stat_orig",
                             pt->table, MUSE_PIXTABLE_STAT);

  /* cut down to nominal wavelength range, that's where it counts! */
  muse_pixtable_restrict_wavelength(pt, 4800., 9300.); /* nominal wl-range */
  /* test-calibrate the table to investigate accuracy of the calibration */
  muse_test_flux_do_calibrate(pt, fluxobj->response, extinction,
                              fluxobj->telluric);
  muse_flux_object *fo = muse_flux_object_new();
  cpl_test(muse_flux_integrate_std(pt, MUSE_FLUX_PROFILE_MOFFAT, fo)
           == CPL_ERROR_NONE);
  /* rename the flux-calibrated columns to be able to run again */
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_DATA, "data_cal");
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_STAT, "stat_cal");
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_DATA,
                             pt->table, "data_orig");
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_STAT,
                             pt->table, "stat_orig");
  /* the resulting response curve should now be (almost) flat *
   * (but don't do the airmass correction again!)             */
  cpl_test(muse_flux_response_compute(fo, MUSE_FLUX_SELECT_BRIGHTEST, 1.0, ref,
                                      NULL, NULL) == CPL_ERROR_NONE);
  cpl_test(muse_flux_get_response_table(fo, MUSE_SPECTRUM_SMOOTH_PPOLY) == CPL_ERROR_NONE);
  cpl_test_nonnull(fo->response);
#if 0
  cpl_error_code ec = cpl_plot_column(NULL, NULL, NULL, fo->response->table,
                                      "lambda", "response");
  printf("plotted: %d\n\n", ec); fflush(stdout);
#endif
  /* do not count the extrapolated regions */
  cpl_table_select_all(fo->response->table);
  cpl_table_and_selected_double(fo->response->table, "lambda", CPL_LESS_THAN, 4800.);
  cpl_table_or_selected_double(fo->response->table, "lambda", CPL_GREATER_THAN, 9300.);
  cpl_table_erase_selected(fo->response->table);
#if 0
  ec = cpl_plot_column(NULL, NULL, NULL, fo->response->table, "lambda",
                       "response");
  printf("plotted: %d\n\n", ec); fflush(stdout);
#endif
  mean = cpl_table_get_column_mean(fo->response->table, "response");
  median = cpl_table_get_column_median(fo->response->table, "response");
  stdev = cpl_table_get_column_stdev(fo->response->table, "response");
  double min = cpl_table_get_column_min(fo->response->table, "response"),
         max = cpl_table_get_column_max(fo->response->table, "response");
  cpl_msg_debug(__func__, "test-calibration statistics: %f (%f) +/- %f, %f ... %f",
                mean, median, stdev, min, max);
  cpl_test(fabs(mean - 43.55) < 0.1); /* XXX why is it not ~50 mag == 1e-20 erg/s/cm**2/Angstrom? */
  cpl_test(fabs(mean - median) < 0.01 && stdev < 0.015);
  cpl_test(max - min < 0.12);
  /* test the same response again, without smoothing this time! */
  muse_table_delete(fo->response);
  fo->response = NULL;
  cpl_table_erase_window(fo->sensitivity, 50, 5);
  cpl_test(muse_flux_get_response_table(fo, MUSE_SPECTRUM_SMOOTH_NONE) == CPL_ERROR_NONE);
  cpl_test_nonnull(fo->response);
  /* results should be roughly similar */
  mean = cpl_table_get_column_mean(fo->response->table, "response");
  median = cpl_table_get_column_median(fo->response->table, "response");
  stdev = cpl_table_get_column_stdev(fo->response->table, "response");
  min = cpl_table_get_column_min(fo->response->table, "response");
  max = cpl_table_get_column_max(fo->response->table, "response");
  cpl_msg_debug(__func__, "test-calibration statistics (unsmoothed): "
                "%f (%f) +/- %f, %f ... %f", mean, median, stdev, min, max);
  cpl_test(fabs(mean - 43.55) < 0.1);
  cpl_test(fabs(mean - median) < 0.01 && stdev < 0.02);
  cpl_test(max - min < 0.25);
  /* test the same response and smoothing again, but with a few rows removed to *
   * trigger the jump-handling in muse_flux_get_response_table_piecewise_poly() */
  muse_table_delete(fo->response);
  fo->response = NULL;
  cpl_table_erase_window(fo->sensitivity, 50, 5);
  cpl_test(muse_flux_get_response_table(fo, MUSE_SPECTRUM_SMOOTH_PPOLY) == CPL_ERROR_NONE);
  cpl_test_nonnull(fo->response);
  /* results should be very similar to the first case */
  mean = cpl_table_get_column_mean(fo->response->table, "response");
  median = cpl_table_get_column_median(fo->response->table, "response");
  stdev = cpl_table_get_column_stdev(fo->response->table, "response");
  min = cpl_table_get_column_min(fo->response->table, "response");
  max = cpl_table_get_column_max(fo->response->table, "response");
  cpl_msg_debug(__func__, "test-calibration statistics (missing rows): "
                "%f (%f) +/- %f, %f ... %f", mean, median, stdev, min, max);
  cpl_test(fabs(mean - 43.55) < 0.1);
  cpl_test(fabs(mean - median) < 0.01 && stdev < 0.015);
  cpl_test(max - min < 0.12);
  muse_flux_object_delete(fo);
  cpl_table_delete(ref);

  /* cut down the pixel table to the innermost area of the standard star PSF, *
   * so that cube reconstruction creates a miniature 3x3xnlambda cube         */
  muse_pixtable_restrict_xpos(pt, -11.12, -9.12); /* center -10.12, 15.08 */
  muse_pixtable_restrict_ypos(pt, 14.08, 16.08);

  /* generate three cubes, with different levels of flux calibration */
  muse_datacube *cubes[3];
  muse_resampling_params *p
    = muse_resampling_params_new(MUSE_RESAMPLE_WEIGHTED_DRIZZLE);
  p->pfx = 1.;
  p->pfy = 1.;
  p->pfl = 1.;

  /* run once without extinction or telluric correction */
  muse_test_flux_do_calibrate(pt, fluxobj->response, NULL, NULL);
  cubes[0] = muse_resampling_cube(pt, p, NULL);
#if 0
  muse_datacube_convert_dq(cubes[0]);
  muse_datacube_save(cubes[0], "cube_0_noext.fits");
#endif
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_DATA, "data_noext");
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_STAT, "stat_noext");

  /* run again with extinction but without telluric correction */
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_DATA,
                             pt->table, "data_orig");
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_STAT,
                             pt->table, "stat_orig");
  muse_test_flux_do_calibrate(pt, fluxobj->response, extinction, NULL);
  cubes[1] = muse_resampling_cube(pt, p, NULL);
#if 0
  muse_datacube_convert_dq(cubes[1]);
  muse_datacube_save(cubes[1], "cube_1_ext.fits");
#endif

  /* test complete failure cases */
  cpl_errorstate ps = cpl_errorstate_get();
  rc = muse_flux_calibrate(NULL, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(ps));
  cpl_errorstate_set(ps);
  rc = muse_flux_calibrate(ptdupe, NULL, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(ps));
  cpl_errorstate_set(ps);
  /* wrong data format */
  cpl_table_set_column_unit(ptdupe->table, MUSE_PIXTABLE_DATA, "adu");
  ps = cpl_errorstate_get();
  rc = muse_flux_calibrate(ptdupe, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_BAD_FILE_FORMAT && !cpl_errorstate_is_equal(ps));
  cpl_errorstate_set(ps);
  cpl_table_set_column_unit(ptdupe->table, MUSE_PIXTABLE_DATA, "count");
  /* the flux calibrated one should fail, too */
  rc = muse_flux_calibrate(pt, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_CONTINUE && !cpl_errorstate_is_equal(ps));
  cpl_errorstate_set(ps);
  /* different INS.MODE */
  char *insmode = cpl_strdup(muse_pfits_get_insmode(ptdupe->header));
  cpl_propertylist_update_string(ptdupe->header, "ESO INS MODE", "NFM-AO-N");
  cpl_propertylist_update_string(fluxobj->response->header, "ESO INS MODE", "WFM-AO-N");
  ps = cpl_errorstate_get();
  rc = muse_flux_calibrate(ptdupe, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(ps);
  cpl_propertylist_update_string(ptdupe->header, "ESO INS MODE", insmode); /* reset */
  cpl_propertylist_update_string(fluxobj->response->header, "ESO INS MODE", insmode);
  cpl_free(insmode);
  /* bad EXPTIME */
  cpl_propertylist_update_double(ptdupe->header, "EXPTIME", 0.); /* bias-like */
  rc = muse_flux_calibrate(ptdupe, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_ILLEGAL_INPUT);
  cpl_propertylist_update_double(ptdupe->header, "EXPTIME", -1.); /* total garbage */
  rc = muse_flux_calibrate(ptdupe, fluxobj->response, extinction, fluxobj->telluric);
  cpl_test(rc == CPL_ERROR_ILLEGAL_INPUT);
  cpl_errorstate_set(ps);
  /* illegal airmass */
  cpl_propertylist_update_double(ptdupe->header, "EXPTIME", 1.); /* reset again */
  cpl_propertylist_update_double(ptdupe->header, "RA", 180.); /* 12h */
  cpl_propertylist_update_double(ptdupe->header, "LST", 10800.); /* 3h */
  cpl_propertylist_erase_regexp(ptdupe->header, "AIRM", 0.); /* remove fallback */
  ps = cpl_errorstate_get();
  cpl_msg_debug("", "_________________ warning below?! ______________________________");
  rc = muse_flux_calibrate(ptdupe, fluxobj->response, extinction, fluxobj->telluric);
  cpl_msg_debug("", "^^^^^^^^^^^^^^^^^ warning above?! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^");
  /* visually check the warning, the airmass function will have given illegal output */
  cpl_test(rc == CPL_ERROR_NONE && !cpl_errorstate_is_equal(ps));
  cpl_errorstate_set(ps);
  cpl_test(muse_pixtable_is_fluxcal(ptdupe));
  muse_pixtable_delete(ptdupe);

  cpl_table_name_column(pt->table, MUSE_PIXTABLE_DATA, "data_ext");
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_STAT, "stat_ext");

  /* run again with extinction and telluric correction */
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_DATA,
                             pt->table, "data_orig");
  cpl_table_duplicate_column(pt->table, MUSE_PIXTABLE_STAT,
                             pt->table, "stat_orig");
  muse_test_flux_do_calibrate(pt, fluxobj->response, extinction,
                              fluxobj->telluric);
  cubes[2] = muse_resampling_cube(pt, p, NULL);
#if 0
  muse_datacube_convert_dq(cubes[2]);
  muse_datacube_save(cubes[2], "cube_2_tell.fits");
#endif
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_DATA, "data_tell");
  cpl_table_name_column(pt->table, MUSE_PIXTABLE_STAT, "stat_tell");

  muse_resampling_params_delete(p);
  muse_test_flux_check_cubes(cubes, 3);

  /* now we have eight relevant columns:                    *
   * data_{orig,noext,ext,tell}, stat_{orig,noext,ext,tell} */
#if 0
  cpl_table_erase_column(pt->table, MUSE_PIXTABLE_XPOS);
  cpl_table_erase_column(pt->table, MUSE_PIXTABLE_YPOS);
  cpl_table_erase_column(pt->table, MUSE_PIXTABLE_DQ);
  cpl_table_erase_column(pt->table, MUSE_PIXTABLE_ORIGIN);
  cpl_table_dump(pt->table, 0, 10, stdout); fflush(stdout);
  cpl_table_dump(pt->table, cpl_table_get_nrow(pt->table)-5, 10, stdout);
  fflush(stdout);
#endif

  /* cleanup */
  muse_pixtable_delete(pt);
  cpl_table_delete(extinction);
  /* test flux object deletion */
  state = cpl_errorstate_get();
  muse_flux_object_delete(fluxobj);
  /* should work without argument */
  muse_flux_object_delete(NULL);
  cpl_test(cpl_errorstate_is_equal(state));

  return cpl_test_end(0);
}
