/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2007-2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#define _XOPEN_SOURCE /* force drand48 definition from stdlib */
#include <stdlib.h>

#include <muse.h>

#define OBJECT_STRING "Test for muse_quality"

/*----------------------------------------------------------------------------*/
/**
  @brief    Test program to check that the functions concerned with quality
            work correctly.

  This program explicitly checks
    muse_quality_is_usable
    muse_quality_set_saturated
    muse_quality_image_reject_using_dq
    muse_quality_dark_badpix
    muse_quality_dark_refine_badpix
    muse_quality_bad_columns
    muse_quality_flat_badpix
    muse_quality_convert_dq
    muse_quality_merge_badpix
  and does minimal (failure) tests on
    muse_quality_merge_badpix_from_file
    muse_quality_copy_badpix_table
 */
/*----------------------------------------------------------------------------*/
int main(int argc, char **argv)
{
  UNUSED_ARGUMENTS(argc, argv);
  cpl_test_init(PACKAGE_BUGREPORT, CPL_MSG_DEBUG);

  /* tests for muse_quality_is_usable(),                 *
   * use the same setup as in muse_resampling_crreject() */
  unsigned badinclude = EURO3D_COSMICRAY;
  cpl_size testcases[] = { 0,
                           0x4240,
                           0x4260,
                           EURO3D_COSMICRAY,
                           EURO3D_COSMICRAY | EURO3D_DARKPIXEL,
                           EURO3D_COSMICRAY | EURO3D_HOTPIXEL,
                           EURO3D_COSMICRAY | EURO3D_LOWQEPIXEL,
                           EURO3D_COSMICRAY | EURO3D_LOWQEPIXEL| EURO3D_DARKPIXEL,
                           EURO3D_HOTPIXEL | EURO3D_DARKPIXEL,
                           EURO3D_HOTPIXEL, EURO3D_DARKPIXEL,
                           -1 };
  cpl_boolean checks[] = { CPL_TRUE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_TRUE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE,
                           CPL_FALSE  /* not reached */ };
  int it = -1; /* index of tests */
  while (testcases[++it] >= 0) {
    int t = testcases[it]; /* shortcut */

    char tflags[33]; /* char array of bit states */
    int i;
    for (i = 31; i >= 0; i--) {
      tflags[i] = (t & 1ul << i) ? '1' : '0';
    } /* for i */
    tflags[32] = '\0'; /* null-terminate the string */

    cpl_msg_info(__func__, "%#010x (%s): %s, %#010x / %s, %#010x / %s",
                 t, tflags, !t ? "not bad" : "    bad",
                 t & badinclude, t & badinclude ? "    include" : "not include",
                 !t ^ badinclude,
                 muse_quality_is_usable(t, badinclude) ? "    xorincl" : "not xorincl");
    cpl_test_eq(muse_quality_is_usable(t, badinclude), checks[it]);
  } /* while valid state */

  /* create a muse_image with content, adapted from muse_image testing */
  int nx = 7, ny = 4;
  muse_image *image = muse_image_new();
  /* add noice in the data extension, of about the typical real bias level */
  image->data = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
  cpl_image_fill_noise_uniform(image->data, 1195., 1205.);
  /* create the variance to go with this noise: *
   * assume GAIN=1.5, RON=3.1, BIAS=1200        */
  image->stat = cpl_image_duplicate(image->data);
  cpl_image_subtract_scalar(image->stat, 1200.);
  cpl_image_divide_scalar(image->stat, 1.5);
  cpl_image_add_scalar(image->stat, 3.1);
  /* set some pixels as bad, in a small checker pattern */
  image->dq = cpl_image_new(nx, ny, CPL_TYPE_INT);
  /* set some saturated pixels */
  cpl_image_set(image->data, 5, 3, 1 << 16);
  cpl_image_set(image->data, 7, 4, (1 << 16) - 1);
  cpl_image_set(image->data, 1, 1, (1 << 16) - 10);
  cpl_image_set(image->data, 4, 3, 0);
  /* and some high but normal ones */
  cpl_image_set(image->data, 1, 2, 2200);
  cpl_image_set(image->data, 1, 3, 21000);
  cpl_image_set(image->data, 1, 4, 41000);
  cpl_test(muse_quality_set_saturated(image) == 4);
  int err;
  cpl_test(cpl_image_get(image->dq, 5, 3, &err) == EURO3D_SATURATED &&
           cpl_image_get(image->dq, 7, 4, &err) == EURO3D_SATURATED &&
           cpl_image_get(image->dq, 1, 1, &err) == EURO3D_SATURATED);
  cpl_test(cpl_image_get(image->dq, 1, 2, &err) == EURO3D_GOODPIXEL &&
           cpl_image_get(image->dq, 1, 3, &err) == EURO3D_GOODPIXEL &&
           cpl_image_get(image->dq, 1, 4, &err) == EURO3D_GOODPIXEL);
  /* test failure cases */
  cpl_errorstate state = cpl_errorstate_get();
  cpl_image_delete(image->stat);
  image->stat = NULL;
  cpl_test(muse_quality_set_saturated(image) == 4); /* stat is irrelevant */
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(muse_quality_set_saturated(NULL) == -1);
  cpl_test(cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_image *imdata = image->data;
  image->data = NULL; /* simulate missing data component */
  cpl_test(muse_quality_set_saturated(image) == -1);
  cpl_test(cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  image->data = imdata;
  cpl_image_delete(image->dq);
  image->dq = NULL; /* simulate missing dq */
  cpl_test(muse_quality_set_saturated(image) == -1);
  cpl_test(cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  muse_image_delete(image);

  /* create another muse_image with content */
  image = muse_image_new();
  /* add noice in the data extension */
  image->data = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
  cpl_image_fill_noise_uniform(image->data, 100, 200);
  /* create the variance to go with this noise: assume GAIN=1.5, RON=3.1, BIAS=100 */
  image->stat = cpl_image_duplicate(image->data);
  cpl_image_subtract_scalar(image->stat, 100);
  cpl_image_divide_scalar(image->stat, 1.5);
  cpl_image_add_scalar(image->stat, 3.1);
  /* set some pixels as bad, in a small checker pattern */
  image->dq = cpl_image_new(nx, ny, CPL_TYPE_INT);
  cpl_image_set(image->dq, 4, 1, EURO3D_COSMICRAY);
  cpl_image_set(image->dq, 4, 3, EURO3D_LOWQEPIXEL);
  cpl_image_set(image->dq, 5, 2, EURO3D_HOTPIXEL);
  cpl_image_set(image->dq, 5, 4, EURO3D_DARKPIXEL);
  cpl_image_set(image->dq, 6, 1, EURO3D_SATURATED);
  cpl_image_set(image->dq, 6, 3, EURO3D_DEADPIXEL);
  const int nbad = 6;
  /* add the headers, and set at least the OBJECT header */
  image->header = cpl_propertylist_new();
  cpl_propertylist_append_string(image->header, "OBJECT", OBJECT_STRING);

  /* check rejecting using DQ */
  muse_image *image2 = muse_image_duplicate(image),
             *image3 = muse_image_duplicate(image);
  state = cpl_errorstate_get();
  cpl_test(muse_quality_image_reject_using_dq(image2->data, image2->dq,
                                              image2->stat) == nbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(cpl_image_count_rejected(image2->data) == nbad);
  /* we test elsewhere that muse_image_reject_from_dq() works, here compare */
  muse_image_reject_from_dq(image);
  cpl_mask *md = cpl_image_get_bpm(image->data),
           *ms = cpl_image_get_bpm(image->stat),
           *m2d = cpl_image_get_bpm(image2->data),
           *m2s = cpl_image_get_bpm(image2->stat);
  cpl_test_eq_mask(md, ms);
  cpl_test_eq_mask(m2d, m2s);
  cpl_test_eq_mask(md, m2d);
  /* should work the same without stat */
  state = cpl_errorstate_get();
  cpl_test(muse_quality_image_reject_using_dq(image3->data, image3->dq, NULL)
           == nbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(cpl_image_count_rejected(image3->data) == nbad);
  cpl_mask *m3d = cpl_image_get_bpm(image3->data);
  cpl_test_eq_mask(md, m3d);
  /* test failure cases */
  state = cpl_errorstate_get();
  cpl_test(muse_quality_image_reject_using_dq(NULL, image2->dq, NULL) == -1);
  cpl_test(cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(muse_quality_image_reject_using_dq(image2->data, NULL, NULL) == -1);
  cpl_test(cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  /* wrong image sizes */
  cpl_image *wrongsize = cpl_image_new(nx - 1, ny + 2, CPL_TYPE_FLOAT);
  cpl_test(muse_quality_image_reject_using_dq(wrongsize, image3->dq, NULL)
           == -2);
  cpl_test(cpl_error_get_code() == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  cpl_test(muse_quality_image_reject_using_dq(image3->data, image3->dq,
                                              wrongsize) == -2);
  cpl_test(cpl_error_get_code() == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_errorstate_set(state);
  /* bad type for DQ */
  cpl_test(muse_quality_image_reject_using_dq(image2->data, image3->stat, NULL)
           == -3);
  cpl_test(cpl_error_get_code() == CPL_ERROR_TYPE_MISMATCH);
  cpl_errorstate_set(state);
  muse_image_delete(image);
  muse_image_delete(image3);
  cpl_image_delete(wrongsize);
  muse_image_delete(image2);

  /* try bad pixel detection on a master dark */
  muse_image *dark = muse_image_load(BASEFILENAME"_dark.fits");
  if (!dark) {
    cpl_msg_error(__func__, "failed to load dark image: %s",
                  cpl_error_get_message());
    return cpl_test_end(1);
  }
  cpl_image *dq = cpl_image_duplicate(dark->dq);
  state = cpl_errorstate_get();
  int ndarkbad = muse_quality_dark_badpix(dark, 0., 5.); /* with std. values */
  cpl_msg_info(__func__, "%d hot pixels found in dark", ndarkbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(ndarkbad == 6287 + 2); /* 2 pixels are marked bad in incoming and are very low */
  /* check some obvious candidates */
  cpl_test(cpl_image_get(dark->dq, 261, 184, &err)
           == EURO3D_HOTPIXEL); /* Q2, 50.0 sigma above bg */
  cpl_test(cpl_image_get(dark->dq, 332, 299, &err)
           == EURO3D_HOTPIXEL); /* Q4, 50.3 sigma */
  cpl_test(cpl_image_get(dark->dq, 8, 6, &err)
           == EURO3D_HOTPIXEL); /* Q1, 22.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 255, 285, &err)
           == EURO3D_HOTPIXEL); /* Q2, 44.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 1023, 1977, &err)
           == (EURO3D_DEADPIXEL | EURO3D_SATURATED | EURO3D_HOTPIXEL)); /* Q4, really bad one in bad column */
  cpl_test(cpl_image_get(dark->dq, 1023, 1949, &err)
           == (EURO3D_DEADPIXEL | EURO3D_HOTPIXEL)); /* Q4,  bad one in bad column */
  cpl_test(cpl_image_get(dark->dq, 333, 300, &err)
           == EURO3D_HOTPIXEL); /* Q4, 8.6 sigma */
  cpl_test(cpl_image_get(dark->dq, 333, 298, &err)
           == EURO3D_GOODPIXEL); /* undetected, Q4, 4.7 sigma */
  /* replace output DQ map with initial DQ and reset CPL bad pixel maps */
  cpl_image_delete(dark->dq);
  dark->dq = cpl_image_duplicate(dq);
  cpl_image_accept_all(dark->data);
  cpl_image_accept_all(dark->stat);
  ndarkbad = muse_quality_dark_badpix(dark, 0., 10.); /* with higher sigma */
  cpl_msg_info(__func__, "%d hot pixels found in dark", ndarkbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(ndarkbad == 5208 + 2);
  cpl_test(cpl_image_get(dark->dq, 261, 184, &err)
           == EURO3D_HOTPIXEL); /* Q2, 50.0 sigma above bg */
  cpl_test(cpl_image_get(dark->dq, 332, 299, &err)
           == EURO3D_HOTPIXEL); /* Q4, 50.3 sigma */
  cpl_test(cpl_image_get(dark->dq, 8, 6, &err)
           == EURO3D_HOTPIXEL); /* Q1, 22.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 255, 285, &err)
           == EURO3D_HOTPIXEL); /* Q2, 44.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 333, 300, &err)
           == EURO3D_GOODPIXEL); /* not detected any more! Q4, 8.6 sigma */
  cpl_test(cpl_image_get(dark->dq, 333, 298, &err)
           == EURO3D_GOODPIXEL); /* undetected, Q4, 4.7 sigma */
  cpl_image_delete(dark->dq);
  dark->dq = cpl_image_duplicate(dq);
  cpl_image_accept_all(dark->data);
  cpl_image_delete(dark->stat); /* should work without stat! */
  dark->stat = NULL;
  ndarkbad = muse_quality_dark_badpix(dark, 10., 5.); /* with dark pixel detection */
  cpl_msg_info(__func__, "%d hot and dark pixels found in dark", ndarkbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(ndarkbad == 6287 + 300); /* the two low pixels are now included in the count */
  cpl_test(cpl_image_get(dark->dq, 261, 184, &err)
           == EURO3D_HOTPIXEL); /* Q2, 50.0 sigma above bg */
  cpl_test(cpl_image_get(dark->dq, 332, 299, &err)
           == EURO3D_HOTPIXEL); /* Q4, 50.3 sigma */
  cpl_test(cpl_image_get(dark->dq, 8, 6, &err)
           == EURO3D_HOTPIXEL); /* Q1, 22.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 255, 285, &err)
           == EURO3D_HOTPIXEL); /* Q2, 44.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 1004, 1979, &err)
           == EURO3D_GOODPIXEL); /* Q4, 5.6 sigma below bg, undetected */
  cpl_test(cpl_image_get(dark->dq, 1011, 1979, &err)
           == EURO3D_GOODPIXEL); /* Q4, 7.1 sigma, undetected */
  cpl_test(cpl_image_get(dark->dq, 1021, 1978, &err)
           == EURO3D_DEADPIXEL); /* Q4, 6123.1 sigma */
  cpl_test(cpl_image_get(dark->dq, 1019, 1975, &err)
           == EURO3D_DEADPIXEL); /* Q4, 21.4 sigma */
  cpl_test(cpl_image_get(dark->dq, 1019, 2030, &err)
           == EURO3D_DEADPIXEL); /* Q4, 12.2 sigma */
  cpl_test(cpl_image_get(dark->dq, 1021, 2133, &err)
           == EURO3D_DEADPIXEL); /* Q4, 15.5 sigma */
  /* also test muse_quality_dark_refine_badpix() with this */
  state = cpl_errorstate_get();
  int sum0 = cpl_image_get_mean_window(dark->dq, 976, 1891, 1023, 2133) * 11664;
  int nrefined1 = muse_quality_dark_refine_badpix(dark, 3., 1);
  int sum1 = cpl_image_get_mean_window(dark->dq, 976, 1891, 1023, 2133) * 11664;
  int nrefined2 = muse_quality_dark_refine_badpix(dark, 3., 2);
  cpl_test(nrefined1 > 0 && cpl_errorstate_is_equal(state));
  cpl_test_zero(nrefined2); /* no new ones with extra iteration */
  /* compared to the previous case, the region with the charge distributed *
   * fromthe bad column should have added ~3700 pixels marked hot (=256)   */
  cpl_test((sum1 - sum0 > 3600 * 256) && (sum1 - sum0 < 3800 * 256));
  cpl_msg_debug(__func__, "%d/%d refined (%d added to summed DQ)", nrefined1,
                nrefined2, sum1 - sum0);
  /* continue with special/failure cases */
  cpl_image_delete(dark->dq);
  dark->dq = cpl_image_duplicate(dq);
  cpl_image_accept_all(dark->data);
  ndarkbad = muse_quality_dark_badpix(dark, 0., 0.); /* no new rejection at all? */
  cpl_msg_info(__func__, "%d hot and dark pixels found in dark", ndarkbad);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(ndarkbad == 544 + 2); /* so many pixels are already rejected and still outside the limits */
  cpl_image *someimage = dark->data;
  dark->data = NULL;
  state = cpl_errorstate_get();
  ndarkbad = muse_quality_dark_badpix(dark, 0., 5.); /* fail without data */
  cpl_test(ndarkbad == -2 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_ILLEGAL_INPUT);
  cpl_msg_debug(__func__, "%d ndarkbad (%d)", ndarkbad, cpl_error_get_code());
  cpl_errorstate_set(state);
  nrefined1 = muse_quality_dark_refine_badpix(dark, 3., 1);
  cpl_msg_debug(__func__, "%d nrefined (%d)", nrefined1, cpl_error_get_code());
  cpl_test(ndarkbad == -2 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_ILLEGAL_INPUT);
  cpl_errorstate_set(state);
  dark->data = someimage;
  cpl_image_delete(dark->dq);
  dark->dq = NULL;
  state = cpl_errorstate_get();
  ndarkbad = muse_quality_dark_badpix(dark, 0., 5.); /* fail without dq */
  cpl_test(ndarkbad == -2 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_ILLEGAL_INPUT);
  cpl_errorstate_set(state);
  nrefined1 = muse_quality_dark_refine_badpix(dark, 3., 1);
  cpl_test(ndarkbad == -2 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_ILLEGAL_INPUT);
  cpl_errorstate_set(state);
  muse_image_delete(dark);
  cpl_image_delete(dq);
  state = cpl_errorstate_get();
  ndarkbad = muse_quality_dark_badpix(NULL, 0., 5.); /* fail without dq */
  cpl_test(ndarkbad == -1 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  nrefined1 = muse_quality_dark_refine_badpix(NULL, 3., 1);
  cpl_test(ndarkbad == -1 && !cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);

  /* test bad column detection on bias data */
  state = cpl_errorstate_get();
  int nbadcol = muse_quality_bad_columns(NULL, 30., 3.); /* everything missing */
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(nbadcol == -1);
  muse_image *bias = muse_image_new();
  nbadcol = muse_quality_bad_columns(bias, 30., 3.); /* missing header */
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(nbadcol == -1);
  bias->header = cpl_propertylist_new();
  cpl_propertylist_append_int(bias->header, "ESO DET BINX", 1);
  cpl_propertylist_append_int(bias->header, "ESO DET BINY", 1);
  unsigned char n;
  for (n = 1; n <= 4; n++) {
    char *kw1 = cpl_sprintf("ESO DET OUT%1hhu NX", n),
         *kw2 = cpl_sprintf("ESO DET OUT%1hhu NY", n),
         *kw3 = cpl_sprintf(QC_BIAS_MASTER_RON, (int)n);
    cpl_propertylist_append_int(bias->header, kw1, 50);
    cpl_propertylist_append_int(bias->header, kw2, 500);
    cpl_propertylist_append_double(bias->header, kw3, 3.);
    cpl_free(kw1);
    cpl_free(kw2);
    cpl_free(kw3);
  } /* for n (four quadrants) */
  nbadcol = muse_quality_bad_columns(bias, 30., 3.); /* missing data */
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(nbadcol == -1);
#define NX 100
#define NY 1000
  bias->data = cpl_image_new(NX, NY, CPL_TYPE_FLOAT);
  cpl_image_fill_noise_uniform(bias->data, 90., 100.);
  nbadcol = muse_quality_bad_columns(bias, 30., 3.); /* missing dq */
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(nbadcol == -1);
  bias->dq = cpl_image_new(NX, NY, CPL_TYPE_INT); /* all good pixels */
  nbadcol = muse_quality_bad_columns(bias, 30., 3.); /* missing stat */
  cpl_test(!cpl_errorstate_is_equal(state)
           && cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test(nbadcol == -1);
  bias->stat = cpl_image_duplicate(bias->data);
  nbadcol = muse_quality_bad_columns(bias, 30., 3.); /* missing stat */
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test(nbadcol == 0);
  /* add 1 to 6 counts to the test image in one column, 15 to 25 in another */
  float *data = cpl_image_get_data_float(bias->data);
  int i = 10, j;
  for (j = 510; j < 950; j++) {
    data[i + j*NX] += drand48() * 5. + 1.;
  } /* for j (vertical pixels) */
  i = 80;
  for (j = 5; j < 400; j++) {
    data[i + j*NX] += drand48() * 10. + 15.;
  } /* for j (vertical pixels) */
  nbadcol = muse_quality_bad_columns(bias, 30., 30.);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_msg_debug(__func__, "nbadcol (with 30./30..) = %d", nbadcol);
  cpl_test(nbadcol > 390 && nbadcol <= 395); /* 395 pixels marked in column 81 */
  nbadcol = muse_quality_bad_columns(bias, 30., 3.);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_msg_debug(__func__, "nbadcol (with 30./3.) = %d", nbadcol);
  cpl_test(nbadcol > 825 && nbadcol <= 1000); /* + 440 pixels in column 11 */
  /* (it may mark more, until the edges of the quadrants) */
  muse_image_delete(bias);

  /* try bad pixel detection on flat-field data */
  muse_image *flat = muse_image_load(BASEFILENAME"_flat.fits");
  cpl_table *trace = cpl_table_load(BASEFILENAME"_trace.fits", 1, 1);
  if (!flat || !trace) {
    cpl_msg_error(__func__, "failed to load flat image or trace table: %s",
                  cpl_error_get_message());
    muse_image_delete(flat);
    cpl_table_delete(trace);
    return cpl_test_end(1);
  }
  double cputime = cpl_test_get_cputime();
  /* use default values from muse_flat */
  int nflatbad = muse_quality_flat_badpix(flat, trace, 5., 5.);
  double diff = cpl_test_get_cputime() - cputime;
  cpl_msg_info(__func__, "badpix searched in %gs", diff);
  cpl_test(nflatbad == 1204 + 167);
  /* really test the DQ extension for expected numbers! */
  nx = cpl_image_get_size_x(flat->dq),
  ny = cpl_image_get_size_y(flat->dq);
  int nd = 0, nh = 0, nl = 0, nb = 0;
  for (i = 1; i <= nx; i++) {
    for (j = 1; j <= ny; j++) {
      if ((unsigned)cpl_image_get(flat->dq, i, j, &err) & EURO3D_DARKPIXEL) {
        nd++;
      }
      if ((unsigned)cpl_image_get(flat->dq, i, j, &err) & EURO3D_HOTPIXEL) {
        nh++;
      }
      if ((unsigned)cpl_image_get(flat->dq, i, j, &err) & EURO3D_LOWQEPIXEL) {
        nl++;
      }
      if ((unsigned)cpl_image_get(flat->dq, i, j, &err) & EURO3D_BADOTHER) {
        nb++;
      }
    } /* for j (vertical direction) */
  } /* for i (horizontal pixels) */
  muse_image_reject_from_dq(flat);
  int nrej = cpl_image_count_rejected(flat->data);
  cpl_msg_debug(__func__, "Tested numbers: %d (%d dark, %d hot, %d lowQE, %d nonpositive)",
                nrej, nd, nh, nl, nb);
  cpl_test(nrej == 1204 + 167 + 439956 - 2); /* two have multiple bad properties */
  cpl_test(nd + nh == nflatbad);
  cpl_test(nd == 1204);
  cpl_test(nh == 167);
  cpl_test(nl == 593);
  cpl_test(nb == 439956);
  cpl_table_delete(trace);
  muse_image_delete(flat);

  /* test muse_quality_convert_dq() */
  dq = cpl_image_new(4000, 4000, CPL_TYPE_INT);
  cpl_image_set(dq, 3, 3, 256);
  cpl_image_set(dq, 8, 2, 512);
  cpl_image_set(dq, 2500, 3000, 128);
  state = cpl_errorstate_get();
  cpl_table *tdq = muse_quality_convert_dq(dq);
  cpl_test_error(muse_cpltable_check(tdq, muse_badpix_table_def));
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_eq(cpl_table_get_nrow(tdq), 3);
#if 0
  cpl_table_dump(tdq, 0, 1000, stdout);
  fflush(stdout);
#endif
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_X, 0, NULL), 3+32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_Y, 0, NULL), 3+32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_DQ, 0, NULL), 256);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_X, 1, NULL), 8+32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_Y, 1, NULL), 2+32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_DQ, 1, NULL), 512);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_X, 2, NULL), 2500+3*32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_Y, 2, NULL), 3000+3*32);
  cpl_test_eq(cpl_table_get_int(tdq, MUSE_BADPIX_DQ, 2, NULL), 128);
  cpl_image_delete(dq);
  /* test failure cases */
  state = cpl_errorstate_get();
  cpl_table *tdq2 = muse_quality_convert_dq(NULL);
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_test_null(tdq2);
  cpl_errorstate_set(state);
  /* test with clean data quality image */
  dq = cpl_image_new(10, 10, CPL_TYPE_INT);
  state = cpl_errorstate_get();
  tdq2 = muse_quality_convert_dq(dq);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_nonnull(tdq2);
  cpl_test_zero(cpl_table_get_nrow(tdq2));
  cpl_image_delete(dq);
  cpl_table_delete(tdq2);

  /* test muse_quality_merge_badpix() */
  /* first, try to merge the table with itself, should be unchanged */
  tdq2 = cpl_table_duplicate(tdq);
  state = cpl_errorstate_get();
  cpl_test(muse_quality_merge_badpix(tdq2, tdq) == CPL_ERROR_NONE);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_zero(cpl_table_compare_structure(tdq, tdq2));
  cpl_test_eq(cpl_table_get_nrow(tdq), cpl_table_get_nrow(tdq2));
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_X, 0, NULL), 3+32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_Y, 0, NULL), 3+32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_DQ, 0, NULL), 256);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_X, 1, NULL), 8+32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_Y, 1, NULL), 2+32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_DQ, 1, NULL), 512);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_X, 2, NULL), 2500+3*32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_Y, 2, NULL), 3000+3*32);
  cpl_test_eq(cpl_table_get_int(tdq2, MUSE_BADPIX_DQ, 2, NULL), 128);
  cpl_table_delete(tdq2);
  /* with with slighly changed table, should create *
   * one additional row and merge two rows into one */
  tdq2 = cpl_table_duplicate(tdq);
  cpl_table_set_int(tdq2, MUSE_BADPIX_X, 0, 10);
  cpl_table_set_int(tdq2, MUSE_BADPIX_DQ, 2, 256);
  cpl_table_set_float(tdq2, MUSE_BADPIX_VALUE, 2, 80.);
#if 0
  cpl_table_dump(tdq, 0, 1000, stdout);
  cpl_table_dump(tdq2, 0, 1000, stdout);
  fflush(stdout);
#endif
  state = cpl_errorstate_get();
  cpl_test(muse_quality_merge_badpix(tdq2, tdq) == CPL_ERROR_NONE);
#if 0
  cpl_table_dump(tdq2, 0, 1000, stdout);
  fflush(stdout);
#endif
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_zero(cpl_table_compare_structure(tdq, tdq2));
  cpl_test_eq(cpl_table_get_nrow(tdq), cpl_table_get_nrow(tdq2) - 1);
  cpl_test_abs(cpl_table_get(tdq2, MUSE_BADPIX_VALUE, 3, NULL), 80., DBL_EPSILON);
  cpl_table_delete(tdq2);
  /* failure cases */
  state = cpl_errorstate_get();
  cpl_test(muse_quality_merge_badpix(tdq, NULL) == CPL_ERROR_NULL_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_test(muse_quality_merge_badpix(NULL, tdq) == CPL_ERROR_NULL_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* try with missing column --> insertion not possible */
  tdq2 = cpl_table_duplicate(tdq);
  cpl_table_erase_column(tdq2, MUSE_BADPIX_DQ);
  state = cpl_errorstate_get();
  cpl_test(muse_quality_merge_badpix(tdq, tdq2) == CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_table_delete(tdq2);

  /* test muse_quality_merge_badpix_from_file() */
  /* XXX add successes */
  /* failure cases */
  int ext;
  state = cpl_errorstate_get();
  cpl_test_null(muse_quality_merge_badpix_from_file(NULL, "blabal.fits",
                                                    "CHAN22", &ext));
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(muse_quality_merge_badpix_from_file(tdq, NULL,
                                                    "CHAN22", &ext));
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  /* bad file name */
  cpl_test_null(muse_quality_merge_badpix_from_file(tdq, "/bla/blabla.fits",
                                                    "CHAN22", &ext));
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND);
  cpl_errorstate_set(state);

  /* test muse_quality_copy_badpix_table() */
  /* XXX add successes */
  /* failure cases */
  state = cpl_errorstate_get();
  cpl_test(muse_quality_copy_badpix_table(NULL, "/bla/bla_out.fits",
                                          1, tdq) == CPL_ERROR_NULL_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_test(muse_quality_copy_badpix_table("/bla/bla_in.fits", NULL,
                                          1, tdq) == CPL_ERROR_NULL_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_test(muse_quality_copy_badpix_table("/bla/bla_in.fits", "/bla/bla_out.fits",
                                          1, NULL) == CPL_ERROR_NULL_INPUT);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_test(muse_quality_copy_badpix_table("/bla/bla_in.fits", "/bla/bla_out.fits",
                                          1, tdq) == CPL_ERROR_FILE_IO);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* bad extension number, currently fails because of missing file */
  cpl_test(muse_quality_copy_badpix_table("/bla/bla_in.fits", "/bla/bla_out.fits",
                                          88, tdq) == CPL_ERROR_FILE_IO);
  cpl_test(!cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);

  cpl_table_delete(tdq);

  return cpl_test_end(0);
}
