/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2014-2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                              Includes                                      *
 *----------------------------------------------------------------------------*/
#include <string.h>

#include <muse.h>
#include "muse_twilight_z.h"

/*---------------------------------------------------------------------------*
 *                              Functions code                               *
 *---------------------------------------------------------------------------*/

/*---------------------------------------------------------------------------*/
/**
  @brief    Add the QC parameters to the combined twilight skyflat
  @param    aImage   the output master image
  @param    aList    the image list of input files
  @return   CPL_ERROR_NONE or another CPL error code on failure
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_twilight_qc_header(muse_image *aImage, muse_imagelist *aList)
{
  cpl_ensure_code(aImage && aList, CPL_ERROR_NULL_INPUT);
  unsigned char ifu = muse_utils_get_ifu(aImage->header);
  cpl_msg_debug(__func__, "Adding QC keywords for IFU %hhu", ifu);

  unsigned stats = CPL_STATS_MEDIAN | CPL_STATS_MEAN | CPL_STATS_STDEV
                 | CPL_STATS_MIN | CPL_STATS_MAX;

  /* write the image statistics for input skyflat field exposures */
  unsigned int k;
  for (k = 0; k < muse_imagelist_get_size(aList); k++) {
    char *keyword = cpl_sprintf(QC_TWILIGHTm_PREFIXi, ifu, k+1);
    muse_basicproc_stats_append_header(muse_imagelist_get(aList, k)->data,
                                       aImage->header, keyword, stats);
    cpl_free(keyword);
    keyword = cpl_sprintf(QC_TWILIGHTm_PREFIXi" "QC_BASIC_NSATURATED, ifu, k+1);
    int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, k)->header,
                                              MUSE_HDR_TMP_NSAT);
    cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
    cpl_free(keyword);
  } /* for k (all images in list) */

  /* properties of the resulting master frame */
  char *kw = cpl_sprintf(QC_TWILIGHTm_MASTER_PREFIX, ifu);
  stats |= CPL_STATS_FLUX;
  muse_basicproc_stats_append_header(aImage->data, aImage->header, kw, stats);
  cpl_free(kw);
  return CPL_ERROR_NONE;
} /* muse_twilight_qc_header() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Reconstruct cube directly from raw exposures, derive relative
            per-IFU fluxes.
  @param    aProcessing  the processing structure
  @param    aParams      the parameters list
  @param    aBPars       basic processing parameters
  @param    aCPars       image combination parameters
  @param    aRPars       resampling parameters
  @return   the reconstructed cube or NULL on error

  This function for each IFU (in parallel) loads the raw data and applies the
  given calibrations using @ref muse_basicproc_load(), using the basic
  processing parameters aBPars. If there were multiple raw inputs, they are
  combined as images using the given combination parameters aCPars, with @ref
  muse_combine_images(). Bad pixels derived from the DQ extension are then
  interpolated, using @cpl_detector_interpolate_rejected(), before a pixel
  table is created for the IFU using @ref muse_pixtable_create().

  @note This function ignores the flat-field spectrum approach of the on-sky
        data (see muse_scibasic, muse_scipost etc.) since the twilight cube
        correction gets applied to the pixel tables before the flat-field
        spectrum is taken out.

  All pixel tables are then restricted to the wavelength range given in aParams,
  and their flux is summed up and recorded in the MUSE_HDR_FLAT_FLUX_SKY"%hhu"
  keyword. This relative flux is then used to scale all pixel tables to a common
  level before they are merged.

  The merged tables are used to recontruct the cube, using @ref
  muse_resampling_cube(). A white-light image is then created in the cube
  structure, using @ref muse_datacube_collapse(), before the cube is returned
  to the caller.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL,
         aProcessing and/or aParams are NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL,
         aParams->lmax is not greater than aParams->lmin}
  @error{set CPL_ERROR_FILE_NOT_FOUND\, return NULL,
         no geometry table could be loaded from aProcessing}
  @error{output warning\. skip this IFU,
         the trace and/or wavelength calibration tables could not be loaded from aProcessing for one IFU}
  @error{output warning\. skip this IFU, basic processing did not work for one IFU}
  @error{propagate error\, return NULL,
         restricting the wavelength range in the first IFU did not work}
  @error{propagate error\, return NULL, reconstructing the cube did not work}
 */
/*----------------------------------------------------------------------------*/
static muse_datacube *
muse_twilight_reconstruct(muse_processing *aProcessing,
                          muse_twilight_params_t *aParams,
                          muse_basicproc_params *aBPars,
                          muse_combinepar *aCPars,
                          muse_resampling_params *aRPars)
{
  cpl_ensure(aProcessing && aParams, CPL_ERROR_NULL_INPUT, NULL);
  double lmin = aParams->lambdamin,
         lmax = aParams->lambdamax;
  cpl_ensure(lmax > lmin, CPL_ERROR_ILLEGAL_INPUT, NULL);
  /* the geometry table is common for all IFUs */
  cpl_table *geo = muse_processing_load_ctable(aProcessing, MUSE_TAG_GEOMETRY_TABLE, 0);
  cpl_ensure(geo, CPL_ERROR_FILE_NOT_FOUND, NULL);

  muse_pixtable **pts = cpl_calloc(kMuseNumIFUs, sizeof(muse_pixtable));
  unsigned char nifu;
  #pragma omp parallel for default(none)                 /* as req. by Ralf */ \
          shared(aBPars, aCPars, aProcessing, geo, pts)
  for (nifu = 1; nifu <= kMuseNumIFUs; nifu++) {
    cpl_table *trace = muse_processing_load_ctable(aProcessing, MUSE_TAG_TRACE_TABLE, nifu),
              *wavecal = muse_processing_load_ctable(aProcessing, MUSE_TAG_WAVECAL_TABLE, nifu);
    if (!trace || !wavecal) {
      cpl_msg_warning(__func__, "Calibrations could not be loaded for IFU "
                      "%2hhu:%s%s", nifu, !trace ? " "MUSE_TAG_TRACE_TABLE : "",
                      !wavecal ? " "MUSE_TAG_WAVECAL_TABLE : "");
      cpl_table_delete(trace);
      cpl_table_delete(wavecal);
      continue; /* skip this IFU */
    }
    /* load and pre-process all raw files */
    cpl_msg_debug(__func__, "load raw files of IFU %2hhu", nifu);
    muse_imagelist *images = muse_basicproc_load(aProcessing, nifu, aBPars);
    if (!images) {
      cpl_msg_warning(__func__, "Basic processing failed for IFU %2hhu: %s",
                      nifu, cpl_error_get_message());
      cpl_table_delete(trace);
      cpl_table_delete(wavecal);
      continue;
    }
    /* search for ILLUM exposure(s) in input list, *
     * prepare them and remove all from the list   */
    cpl_table *tattached = muse_basicproc_get_illum(images, trace, wavecal, geo);
    /* combine the raw exposures into a master image */
    cpl_msg_debug(__func__, "combine raw files of IFU %2hhu", nifu);
    muse_image *masterimage = muse_combine_images(aCPars, images);
    muse_twilight_qc_header(masterimage, images);
    muse_imagelist_delete(images);
    if (!masterimage) {
      cpl_msg_warning(__func__, "Combining individual images failed for IFU %2hhu: %s",
                      nifu, cpl_error_get_message());
      cpl_table_delete(tattached);
      cpl_table_delete(trace);
      cpl_table_delete(wavecal);
      continue;
    }

    /* apply bad-pixel interpolation on this image, so that the *
     * per-IFU integrated fluxes are independent of bad pixels  */
    int nbad = muse_quality_image_reject_using_dq(masterimage->data,
                                                  masterimage->dq,
                                                  masterimage->stat);
    cpl_detector_interpolate_rejected(masterimage->data);
    /* doing this for the variance is ugly but we cannot do much else */
    cpl_detector_interpolate_rejected(masterimage->stat);
    /* still leave them marked in the DQ extension */
    cpl_msg_debug(__func__, "interpolated over %d bad pixels in IFU %2hhu",
                  nbad, nifu);

    /* create pixel table for this master image */
    cpl_msg_debug(__func__, "create pixel table for IFU %2hhu", nifu);
    pts[nifu - 1] = muse_pixtable_create(masterimage, trace, wavecal, geo);
    cpl_table_delete(trace);
    cpl_table_delete(wavecal);
    /* propagate the QC parameters from the image header */
    if (pts[nifu - 1]) {
      cpl_propertylist_copy_property_regexp(pts[nifu - 1]->header,
                                            masterimage->header,
                                            QC_TWILIGHT_REGEXP, 0);
      if (tattached) { /* test before, to not throw an error */
        muse_basicproc_apply_illum(pts[nifu - 1], tattached);
      } /* if attached flat given */
    } /* if pixel table created */
    muse_image_delete(masterimage);
    cpl_table_delete(tattached);

    /* in AO mode we also need to mask the NaD notch filter range */
    if (muse_pfits_get_mode(pts[nifu - 1]->header) > MUSE_MODE_WFM_NONAO_N) {
      muse_basicproc_mask_notch_filter(pts[nifu - 1], nifu);
    } /* if AO mode */
  } /* for nifu (all IFUs) */
  cpl_table_delete(geo);

  muse_pixtable *pt = pts[0];
  muse_pixtable_restrict_wavelength(pt, lmin, lmax);
  if (muse_pixtable_get_nrow(pt) < 1) {
    cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT,
                          "After cutting to %.2f..%.2f Angstrom in wavelength "
                          "no pixels for cube reconstruction are left!", lmin,
                          lmax);
    for (nifu = 1; nifu <= kMuseNumIFUs; nifu++) {
      muse_pixtable_delete(pts[nifu - 1]);
    } /* for nifu (all IFUs) */
    cpl_free(pts);
    return NULL;
  } /* if */

  cpl_array *scales = NULL;
  if (getenv("MUSE_TWILIGHT_SCALES")) {
    scales = muse_cplarray_new_from_delimited_string(getenv("MUSE_TWILIGHT_SCALES"), ",");
    double scale = atof(cpl_array_get_string(scales, 0));
    cpl_table_divide_scalar(pt->table, MUSE_PIXTABLE_DATA, scale);
    cpl_table_divide_scalar(pt->table, MUSE_PIXTABLE_STAT, scale*scale);
    int nscales = cpl_array_get_size(scales);
    if (nscales != kMuseNumIFUs) {
      cpl_msg_warning(__func__, "%d external scales found instead of %d! Not "
                      "using them.", nscales, kMuseNumIFUs);
      cpl_array_delete(scales);
      scales = NULL;
    } else {
      cpl_msg_info(__func__, "Using %d external scales.", nscales);
    }
#if 1
    cpl_array_dump(scales, 0, 30, stdout);
    fflush(stdout);
#endif
  } /* if */

  double fluxref = cpl_table_get_column_mean(pt->table, MUSE_PIXTABLE_DATA)
                 * muse_pixtable_get_nrow(pt);
  cpl_msg_debug(__func__, "pixel table of IFU  1, fluxref = %e, %"CPL_SIZE_FORMAT
                " rows", fluxref, muse_pixtable_get_nrow(pt));
  /* merge all pixel tables a la muse_pixtable_load_merge_channels() *
   * but first restricting the wavelength range and with flux        *
   * correction using the data we have in the table                  */
  cpl_propertylist_append_double(pt->header, MUSE_HDR_FLAT_FLUX_SKY"1", fluxref);
  char *kw = cpl_sprintf(QC_TWILIGHTm_INTFLUX, (unsigned char)1);
  cpl_propertylist_append_float(pt->header, kw, fluxref);
  cpl_free(kw);
  for (nifu = 2; nifu <= kMuseNumIFUs; nifu++) {
    if (!pts[nifu - 1]) {
      continue;
    } /* if */
    muse_pixtable_restrict_wavelength(pts[nifu - 1], lmin, lmax);
    if (muse_pixtable_get_nrow(pts[nifu - 1]) < 1) {
      continue;
    } /* if */
    double flux = cpl_table_get_column_mean(pts[nifu - 1]->table, MUSE_PIXTABLE_DATA)
                * muse_pixtable_get_nrow(pts[nifu - 1]),
           scale = 1.;
    if (nifu == 24 &&
        muse_pfits_get_mode(pts[nifu - 1]->header) == MUSE_MODE_NFM_AO_N) {
      /* CHAN24 in NFM is vignetted, special handling */
      kw = cpl_sprintf(MUSE_HDR_FLAT_FLUX_SKY"%hhu", 23);
      double flux23 = cpl_propertylist_get_double(pt->header, kw);
      cpl_free(kw);
      cpl_msg_debug(__func__, "NFM: using flux (%.3g) from CHAN23 for "
                    "normalization (instead of %.3g)", flux23, flux);
      flux = flux23;
    } /* if IFU 24 and NFM */
    if (scales) {
      scale = atof(cpl_array_get_string(scales, nifu - 1));
      /* also recomputed the flux that is then saved in the header */
      flux = scale * fluxref;
    } else {
      scale = flux / fluxref;
    } /* else */
    kw = cpl_sprintf(MUSE_HDR_FLAT_FLUX_SKY"%hhu", nifu);
    cpl_propertylist_append_double(pt->header, kw, flux);
    cpl_free(kw);
    cpl_msg_debug(__func__, "merging pixel table of IFU %2hhu, flux = %e, scale = %f",
                  nifu, flux, scale);
    cpl_table_divide_scalar(pts[nifu - 1]->table, MUSE_PIXTABLE_DATA, scale);
    cpl_table_divide_scalar(pts[nifu - 1]->table, MUSE_PIXTABLE_STAT, scale*scale);
    cpl_table_insert(pt->table, pts[nifu - 1]->table, muse_pixtable_get_nrow(pt));
    /* again, propagate the QC parameters to the merged pixel table */
    cpl_propertylist_copy_property_regexp(pt->header, pts[nifu - 1]->header,
                                          QC_TWILIGHT_REGEXP, 0);
    /* add the pixel-table based flux as QC as well */
    kw = cpl_sprintf(QC_TWILIGHTm_INTFLUX, nifu);
    cpl_propertylist_append_float(pt->header, kw, flux);
    cpl_free(kw);
    muse_pixtable_delete(pts[nifu - 1]);
  } /* for nifu (all IFUs) */
  cpl_free(pts);
  /* reset limits in the header, necessary due to manual operation on table data */
  muse_pixtable_compute_limits(pt);
  cpl_array_delete(scales);

  /* now just reconstruct the cube and collapse over the full range */
  muse_datacube *cube = muse_resampling_cube(pt, aRPars, NULL);
  muse_pixtable_delete(pt);
  if (!cube) {
    return NULL;
  }
  /* also create a corresponding white-light image */
  muse_image *white = muse_datacube_collapse(cube, NULL);
  cube->recimages = muse_imagelist_new();
  cube->recnames = cpl_array_new(1, CPL_TYPE_STRING);
  muse_imagelist_set(cube->recimages, white, 0);
  cpl_array_set_string(cube->recnames, 0, "white");

  /* want NANs instead of zeros, and save space */
  muse_datacube_convert_dq(cube);

  return cube;
} /* muse_twilight_reconstruct() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Filter an image, normalize it, and then fit a 2D polynomial.
  @param    aImage   the image to process
  @param    aXOrder   polynomial order in x direction
  @param    aYOrder   polynomial order in y direction
  @param    aBPM      bad pixel mask, area to except from filtering and
                      polynomial fit
  @param    aVign     vignetting mask, area to except from polynomial fit
  @return   the image created from the 2D polynomial or NULL on error

  This function uses a hardcoded 5x7 median filter with cpl_image_filter_mask()
  to smooth an image, across the horizontal edges between the slicer stacks and
  the vertical edges due to uneven illumination of the edge slices of each IFU.
  The area masked by aBPM and and pixels with infinite or zero values are
  ignored by this filtering.
  The filtered image is then normalized, again ignoring the masked region.
  Then, the aVign mask is added to the ignored area, before the image is given
  to @ref muse_utils_image_fit_polynomial() for the polynomial fit. The output
  image is then normalized again, ignoring the area given by the merged masks.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL,
         aImage or its data component are NULL}
 */
/*----------------------------------------------------------------------------*/
static cpl_image *
muse_twilight_image_normalize_and_fit(muse_image *aImage,
                                      unsigned int aXOrder,
                                      unsigned int aYOrder,
                                      const cpl_mask *aBPM,
                                      const cpl_mask *aVign)
{
  cpl_ensure(aImage && aImage->data, CPL_ERROR_NULL_INPUT, NULL);

  /* reject values from the white-light mask, and extra infinite pixels */
  if (aBPM) {
    cpl_image_reject_from_mask(aImage->data, aBPM);
  }
  cpl_image_reject_value(aImage->data, CPL_VALUE_NOTFINITE | CPL_VALUE_ZERO);

  /* filter the image to remove outliers (like strongly deviant slices) */
  /* use a 5x7 median filter */
  cpl_image *filtered = cpl_image_duplicate(aImage->data);
  cpl_mask *mask = cpl_mask_new(5, 7);
  cpl_mask_not(mask);
  cpl_errorstate prestate = cpl_errorstate_get();
  cpl_image_filter_mask(filtered, aImage->data, mask, CPL_FILTER_MEDIAN,
                        CPL_BORDER_FILTER);
  cpl_mask_delete(mask);
  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_msg_warning(__func__, "filtering cube plane failed: %s",
                    cpl_error_get_message());
  }

  /* compute the mean of the good pixels and use that to normalize the image */
  double mean = cpl_image_get_mean(filtered);
  cpl_msg_debug(__func__, "mean = %f", mean);
  cpl_image_multiply_scalar(filtered, 1. / mean);

  /* mask out the vignetted area (if given), before applying the fit */
  cpl_mask *bpm = cpl_image_get_bpm(filtered);
  if (aVign) {
    cpl_mask_or(bpm, aVign);
  }
  cpl_image *fit = muse_utils_image_fit_polynomial(filtered,
                                                   aXOrder, aYOrder);
  /* normalize again */
  mean = cpl_image_get_mean(fit);
  cpl_msg_debug(__func__, "mean(fit) = %f", mean);
  cpl_image_multiply_scalar(fit, 1. / mean);
  cpl_image_delete(filtered);
  return fit;
} /* muse_twilight_image_normalize_and_fit() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Interpret the command line options and execute the data processing
  @param    aProcessing   the processing structure
  @param    aParams       the parameters list
  @return   0 if everything is ok, -1 something went wrong
 */
/*---------------------------------------------------------------------------*/
int
muse_twilight_compute(muse_processing *aProcessing,
                      muse_twilight_params_t *aParams)
{
  muse_datacube *cube = NULL,
                *cubefit = cpl_calloc(1, sizeof(muse_datacube));
  cubefit->data = cpl_imagelist_new();
  if (getenv("MUSE_TWILIGHT_SKIP")) {
    char *fn = getenv("MUSE_TWILIGHT_SKIP");
    cube = muse_datacube_load(fn);
    cpl_msg_warning(__func__, "Skipping active: loaded previously generated "
                    "cube from \"%s\"", fn);
    /* ensure that the first collapse image is "white" */
    if (!cube->recimages) { /* no reconstructed images at all */
      /* create the white-light image */
      cube->recimages = muse_imagelist_new();
      cube->recnames = cpl_array_new(1, CPL_TYPE_STRING);
      muse_image *white = muse_datacube_collapse(cube, NULL);
      muse_imagelist_set(cube->recimages, white, 0);
      cpl_array_set_string(cube->recnames, 0, "white");
    } else if (!cube->recnames ||
               (cube->recnames &&
                strncmp("white", cpl_array_get_string(cube->recnames, 0), 6))) {
      /* replace first reconstructed image with white-light */
      muse_image *white = muse_datacube_collapse(cube, NULL);
      muse_imagelist_set(cube->recimages, white, 0);
      cpl_array_set_string(cube->recnames, 0, "white");
    } /* else */

    /* add something to the usedframes in aProcessing for saving below to work */
    cpl_frameset *frames = muse_frameset_find_tags(aProcessing->inframes,
                                                   aProcessing->intags, -1, 0);
    cpl_size iframe, nframes = cpl_frameset_get_size(frames);
    for (iframe = 0; iframe < nframes; iframe++) {
      cpl_frame *frame = cpl_frameset_get_position(frames, iframe);
      muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_RAW, 1);
    } /* for iframe (all raw inputs) */
    cpl_frameset_delete(frames);
    cpl_msg_warning(__func__, "Skipping active: faked used frames using only "
                    "raw inputs:");
    cpl_frameset_dump(aProcessing->usedframes, stdout);
    fflush(stdout);
    cubefit->header = cpl_propertylist_duplicate(cube->header);
  } /* if skip */

  cpl_error_code rc = CPL_ERROR_NONE;
  if (!cube) { /* skipping did not happen */
    muse_basicproc_params *bpars = muse_basicproc_params_new(aProcessing->parameters,
                                                             "muse.muse_twilight");
    muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
                                                 "muse.muse_twilight");
    muse_resampling_type resample
      = muse_postproc_get_resampling_type(aParams->resample_s);
    muse_resampling_params *rp = muse_resampling_params_new(resample);
    rp->crtype = muse_postproc_get_cr_type(aParams->crtype_s);
    rp->crsigma = aParams->crsigma;
    rp->dlambda = aParams->dlambda;
    rp->pfx = 1.;
    rp->pfy = 1.;
    rp->pfl = 1.;
    cube = muse_twilight_reconstruct(aProcessing, aParams, bpars, cpars, rp);
    muse_basicproc_params_delete(bpars);
    muse_combinepar_delete(cpars);
    muse_resampling_params_delete(rp);
    if (!cube) {
      muse_datacube_delete(cubefit);
      return -1;
    }
    /* duplicate the cube header and its WCS before saving destroys the WCS */
    cubefit->header = cpl_propertylist_duplicate(cube->header);
    rc = muse_processing_save_cube(aProcessing, -1, cube,
                                   MUSE_TAG_CUBE_SKYFLAT,
                                   MUSE_CUBE_TYPE_FITS);
    /* the QC parameters are saved with the unprocessed cube, now delete them */
    cpl_propertylist_erase_regexp(cubefit->header, QC_TWILIGHT_REGEXP, 0);
  } /* if no skipping */

  /* now mark empty areas in the white-light image as bad; just   *
   * access the first reconstructed image, since the white-light  *
   * image is the only one created by muse_twilight_reconstruct() */
  muse_image *wmuse = muse_imagelist_get(cube->recimages, 0);
  if (!wmuse) {
    muse_datacube_delete(cube);
    cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT,
                          "White-light image is missing!");
    muse_datacube_delete(cubefit);
    return -1;
  }
  cpl_image *white = cpl_image_duplicate(wmuse->data);
  cpl_image_reject_value(white, CPL_VALUE_NOTFINITE | CPL_VALUE_ZERO);
  cpl_mask *bpm = cpl_image_get_bpm(white),
           *bpm2 = cpl_mask_duplicate(bpm);
  /* force a rejection of the vertical border pixels */
  cpl_mask *border = cpl_mask_new(1, cpl_mask_get_size_y(bpm2));
  cpl_mask_not(border);
  cpl_mask_copy(bpm2, border, 1, 1);
  cpl_mask_copy(bpm2, border, cpl_mask_get_size_x(bpm2), 1);
  cpl_mask_delete(border);
  /* use dilation to make the mask wider, i.e. to *
   * exclude more of the (usually bad) border     */
  cpl_mask *kernel = cpl_mask_new(5, 1);
  cpl_mask_not(kernel);
  cpl_mask_filter(bpm2, bpm, kernel, CPL_FILTER_DILATION, CPL_BORDER_NOP);
  cpl_mask_delete(kernel);
  bpm = cpl_image_set_bpm(white, bpm2);
  cpl_mask_delete(bpm);
  bpm = bpm2; /* easier handle to remember */

  /* boolean value for the principle instrument mode, since that   *
   * determines how we treat the vignetting (positive or negative) */
  cpl_boolean isWFM = muse_pfits_get_mode(cube->header) != MUSE_MODE_NFM_AO_N;

  /* get or create a vignetting mask */
  muse_mask *maskimage = NULL;
  if (isWFM) {
    cpl_msg_debug(__func__, "WFM: use given %s, if any...", MUSE_TAG_VIGN_MASK);
    /* apply a mask, if one was given, on top of the one that the image already has */
    maskimage = muse_processing_load_mask(aProcessing, MUSE_TAG_VIGN_MASK);
    if (maskimage) {
      /* adapt the mask to the size of the actual cube */
      cpl_mask *mask = muse_cplmask_adapt_to_image(maskimage->mask,
                                                   cpl_imagelist_get(cube->data, 0));
      cpl_mask_delete(maskimage->mask);
      maskimage->mask = mask;
    }
  } else {
    /* for NFM, create an internal mask that fills the whole width of *
     * the cube and the top nmasktop pixels (default: 22 ~ 1.8 IFUs)  */
    cpl_msg_info(__func__, "NFM: creating internal %s...", MUSE_TAG_VIGN_MASK);
    maskimage = muse_mask_new();
    int nx = cpl_image_get_size_x(white),
        ny = cpl_image_get_size_y(white),
        nmasktop = aParams->vignnfmmask > 0 && aParams->vignnfmmask < ny
                 ? aParams->vignnfmmask : 22;
    maskimage->header = cpl_propertylist_duplicate(cube->header);
    maskimage->mask = cpl_mask_new(nx, ny);
    /* mark the top pixels as masked */
    int j;
    for (j = ny - nmasktop; j <= ny; j++) {
      int i;
      for (i = 1; i <= nx; i++) {
        cpl_mask_set(maskimage->mask, i, j, CPL_BINARY_1);
      } /* for i (all mask columns) */
    } /* for j (all top mask pixels) */
  }
  /* and we can normalize, fit, and then normalize again, every plane in the cube */
  muse_image *image = muse_image_new();
  int ipl, npl = cpl_imagelist_get_size(cube->data);
  for (ipl = 0; ipl < npl; ipl++) {
    image->data = cpl_imagelist_get(cube->data, ipl);
    image->stat = cpl_imagelist_get(cube->stat, ipl);
    cpl_mask *vignmask = maskimage ? maskimage->mask : NULL;
    cpl_image *fit = muse_twilight_image_normalize_and_fit(image,
                                                           aParams->xorder,
                                                           aParams->yorder,
                                                           bpm, vignmask);
    cpl_imagelist_set(cubefit->data, fit, ipl);
  } /* for ipl (all cube image planes) */
  image->data = NULL;
  image->stat = NULL;
  muse_image_delete(image);
  muse_datacube_delete(cube);

#if 0 /* extra debug output for the globally fitted cube, without vignetted area */
  rc = muse_processing_save_cube(aProcessing, -1, cubefit,
                                 "TWILIGHT_CUBE_FIT", MUSE_CUBE_TYPE_FITS);
#endif

  muse_image *whitefit = muse_datacube_collapse(cubefit, NULL);
  cpl_image_delete(whitefit->dq);
  whitefit->dq = NULL;
  cpl_propertylist_update_string(whitefit->header, "OBJECT",
                                 "white from global fit");
  cubefit->recimages = muse_imagelist_new();
  muse_imagelist_set(cubefit->recimages, whitefit, 0);
  cubefit->recnames = cpl_array_new(1, CPL_TYPE_STRING);
  cpl_array_set_string(cubefit->recnames, 0, "white_global_fit");

  /* normalize the non-smoothed white-light image and divide it by *
   * the smoothed version; then fit the masked (vignetted) area    */
  double mean = cpl_image_get_mean(white);
  cpl_image_divide_scalar(white, mean);
  cpl_image *wdiv = cpl_image_divide_create(white, whitefit->data);
  /* append to reconstructed images */
  image = muse_image_new();
  image->header = cpl_propertylist_new();
  cpl_propertylist_append_string(image->header, "OBJECT", "normalized "
                                 "white-light divided by white global fit");
  cpl_propertylist_append_string(image->header, "BUNIT", "1"); /* unitless */
  image->data = wdiv;
  image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(wdiv));
  muse_imagelist_set(cubefit->recimages, image,
                     muse_imagelist_get_size(cubefit->recimages));
  cpl_array_set_size(cubefit->recnames,
                     cpl_array_get_size(cubefit->recnames) + 1);
  cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
                       "white_div_global");
  /* keep mask due to non-rectangular FOV around */
  cpl_mask *fovmask = cpl_mask_duplicate(cpl_image_get_bpm(white));
  if (maskimage) {
    bpm = cpl_image_get_bpm(white);
    cpl_mask_or(bpm, maskimage->mask);
  } /* if maskimage */
  cpl_image_delete(white);

  /* now invert the mask image (we need to fit the region that it contains!) */
  if (maskimage) {
    /* create mask of only the vignetted area */
    cpl_mask *vignmask = cpl_mask_duplicate(fovmask);
    cpl_mask_not(maskimage->mask);
    cpl_mask_or(vignmask, maskimage->mask);
    cpl_image *wdiv2 = cpl_image_duplicate(wdiv);
    cpl_image_reject_from_mask(wdiv2, vignmask);

    /* search for pixels strongly deviating from unity and reject them */
    if (isWFM && aParams->vignmaskedges > 0.) {
      cpl_msg_info(__func__, "Excluding strong edges (stronger than %.1f%%) "
                   "from the vignetted area", aParams->vignmaskedges * 100.);
      cpl_image *wdivX = cpl_image_duplicate(wdiv2);
      cpl_mask *wdivXmask = cpl_mask_duplicate(cpl_image_get_bpm(wdivX));
      cpl_image_fill_rejected(wdivX, 1.);
      cpl_image_accept_all(wdivX);
      int i, nx = cpl_image_get_size_x(wdivX),
          j, ny = cpl_image_get_size_y(wdivX);
      /* first column-wise */
      for (i = 1; i <= nx; i++) {
        for (j = 2; j <= ny; j++) {
          int err;
          double v1 = cpl_image_get(wdivX, i, j - 1, &err),
                 v2 = cpl_image_get(wdivX, i, j, &err);
          if (fabs(v2 - v1) < aParams->vignmaskedges) {
            continue;
          }
          /* large difference found */
          cpl_msg_debug(__func__, "%03d %03d v1 %f V2 %f --> delta %f",
                        i, j, v1, v2, fabs(v2 - v1));
          double dv1 = fabs(v1 - 1.),
                 dv2 = fabs(v2 - 1.);
          if (dv1 > dv2) {
            cpl_mask_set(wdivXmask, i, j - 1, CPL_BINARY_1);
          } else {
            cpl_mask_set(wdivXmask, i, j, CPL_BINARY_1);
          }
        }
      }
      /* now row-wise */
      for (j = 1; j <= ny; j++) {
        for (i = 2; i <= nx; i++) {
          int err;
          double v1 = cpl_image_get(wdivX, i - 1, j, &err),
                 v2 = cpl_image_get(wdivX, i, j, &err);
          if (fabs(v2 - v1) < aParams->vignmaskedges) {
            continue;
          }
          /* large difference found */
          cpl_msg_debug(__func__, "%03d %03d v1 %f V2 %f --> delta %f",
                        i, j, v1, v2, fabs(v2 - v1));
          double dv1 = fabs(v1 - 1.),
                 dv2 = fabs(v2 - 1.);
          if (dv1 > dv2) {
            cpl_mask_set(wdivXmask, i - 1, j, CPL_BINARY_1);
          } else {
            cpl_mask_set(wdivXmask, i, j, CPL_BINARY_1);
          }
        }
      }
      /* transfer the edited mask to the divided image for the smoothing */
      cpl_image_reject_from_mask(wdiv2, wdivXmask);

      /* append to reconstructed images, before throwing away */
      image = muse_image_new();
      image->header = cpl_propertylist_new();
      cpl_propertylist_append_string(image->header, "OBJECT",
                                     "white-light divided by white global, mask excl edges");
      cpl_propertylist_append_string(image->header, "BUNIT", "1"); /* unitless */
      image->data = wdivX;
      image->dq = cpl_image_new_from_mask(wdivXmask);
      cpl_mask_delete(wdivXmask);
      muse_imagelist_set(cubefit->recimages, image,
                         muse_imagelist_get_size(cubefit->recimages));
      cpl_array_set_size(cubefit->recnames,
                         cpl_array_get_size(cubefit->recnames) + 1);
      cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
                           "white_div_global_excl_edges");
    } /* if vignmaskedges positive */

    /* do the smoothing in one of the several possible ways */
    cpl_image *fit = NULL;
    if (aParams->vignsmooth == MUSE_TWILIGHT_PARAM_VIGNSMOOTH_GAUSSIAN) {
      /* possibly reset to defaults (see muse_twilight.xml) */
      if (aParams->vignxpar < 0.) {
        aParams->vignxpar = 10;
      }
      if (aParams->vignypar < 0.) {
        aParams->vignypar = 10;
      }
      /* use Gaussian filter to smooth the vignetted area */
      double fwhm = ((double)aParams->vignxpar + (double)aParams->vignypar) / 2.;
      cpl_msg_info(__func__, "Running %.1f pix FWHM Gaussian filter...", fwhm);
      fit = cpl_image_duplicate(wdiv2);
      /* create odd halfwidth of about fwhm */
      int hw = lround(fwhm);
      hw = hw % 2 ? hw : hw + 1;
      cpl_msg_debug(__func__, "using width 2 x %d + 1 for Gaussian matrix", hw);
      cpl_matrix *gauss = muse_matrix_new_gaussian_2d(hw, hw,
                                                      fwhm * CPL_MATH_SIG_FWHM);
      cpl_errorstate prestate = cpl_errorstate_get();
      cpl_image_filter(fit, wdiv2, gauss, CPL_FILTER_LINEAR, CPL_BORDER_FILTER);
      cpl_matrix_delete(gauss);
      cpl_image_reject_from_mask(fit, cpl_image_get_bpm(wdiv2)); /* re-reject! */
      if (isWFM) {
        /* WFM had vignetting in lamp-flats, so valid values are > 1 */
        cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
      } else {
        /* NFM had vignetting in sky exposures, valid values are < 1 */
        cpl_image_threshold(fit, -FLT_MAX, 1., -FLT_MAX, 1.);
      }
      if (!cpl_errorstate_is_equal(prestate)) {
        cpl_msg_warning(__func__, "filtering vignetted area failed: %s",
                        cpl_error_get_message());
      }
    } else if (aParams->vignsmooth == MUSE_TWILIGHT_PARAM_VIGNSMOOTH_MEDIAN) {
      /* possibly reset to defaults (see muse_twilight.xml) */
      if (aParams->vignxpar < 0.) {
        aParams->vignxpar = 5;
      }
      if (aParams->vignypar < 0.) {
        aParams->vignypar = 5;
      }
      cpl_msg_info(__func__, "Running %dx%d median filter...",
                   aParams->vignxpar, aParams->vignypar);
      fit = cpl_image_duplicate(wdiv2);
      cpl_mask *mask = cpl_mask_new(aParams->vignxpar, aParams->vignypar);
      cpl_mask_not(mask);
      cpl_errorstate prestate = cpl_errorstate_get();
      cpl_image_filter_mask(fit, wdiv2, mask, CPL_FILTER_MEDIAN,
                            CPL_BORDER_FILTER);
      cpl_mask_delete(mask);
      if (isWFM) {
        /* again, replace values less than 1 */
        cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
      } else {
        cpl_image_threshold(fit, -FLT_MAX, 1., -FLT_MAX, 1.);
      }
      if (!cpl_errorstate_is_equal(prestate)) {
        cpl_msg_warning(__func__, "filtering vignetted ared failed: %s",
                        cpl_error_get_message());
      }
    } else { /* MUSE_TWILIGHT_PARAM_VIGNSMOOTH_POLYFIT */
      if (aParams->vignxpar < 0.) {
        aParams->vignxpar = 4;
      }
      if (aParams->vignypar < 0.) {
        aParams->vignypar = 4;
      }
      cpl_msg_info(__func__, "Running polyfit (orders %dx%d) ...",
                   aParams->vignxpar, aParams->vignypar);
      /* since we want to correct only vignetting, everything in this image *
       * should be one or above (for WFM) or one or below (for NFM), so     *
       * that we should unlikely values                                     */
      cpl_image *wdiv3 = cpl_image_duplicate(wdiv2);
      if (isWFM) {
        cpl_image_threshold(wdiv3, 1., FLT_MAX, 1., FLT_MAX);
      } else {
        cpl_image_threshold(wdiv3, -FLT_MAX, 1., -FLT_MAX, 1.);
      }

      /* do the fit */
      fit = muse_utils_image_fit_polynomial(wdiv3, aParams->vignxpar,
                                            aParams->vignypar);
      cpl_image_delete(wdiv3);
      if (isWFM) {
        cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
      } else {
        cpl_image_threshold(fit, -FLT_MAX, 1., -FLT_MAX, 1.);
      }
    } /* else */

    if (isWFM) {
      /* set the original rejection mask again */
      cpl_image_reject_from_mask(fit, vignmask);
    } else {
      /* for NFM, reject using the originally created mask, but check *
       * row levels before doing so: there might be a row below 1 at  *
       * the bottom of the fit region                                 */
      cpl_mask *mask = cpl_mask_duplicate(maskimage->mask);
      /* the mask currently marks the non-vignetted area, so invert it */
      cpl_mask_not(mask);
      int j, nx = cpl_mask_get_size_x(mask),
          ny = cpl_mask_get_size_y(mask);
      double maxval = -DBL_MAX;
      for (j = ny; j >= 1; j--) {
        int nmasked = cpl_mask_count_window(mask, 1, j, nx, j);
        if (nmasked <= 0) {
          continue;
        } /* if */
        double avg = cpl_image_get_mean_window(fit, 1, j, nx, j),
               stdev = cpl_image_get_stdev_window(fit, 1, j, nx, j),
               median = cpl_image_get_median_window(fit, 1, j, nx, j);
        cpl_msg_debug(__func__, "row %d (%d): %.5f +/- %.5f (%.5f)", j, nmasked,
                      avg, stdev, median);
        if (maxval < avg) {
          maxval = avg;
        } /* if */
        if (fabs(maxval - 1.) < 0.1 && avg < maxval) {
          cpl_msg_debug(__func__, "    (below max, removing from mask)");
          int i;
          for (i = 1; i <= nx; i++) {
            cpl_mask_set(mask, i, j, CPL_BINARY_0);
          } /* for i (all mask columns) */
        } /* if */
      } /* for j (all mask rows) */
      /* now we again need the non-vignetted area, so invert it */
      cpl_mask_not(mask);
      cpl_image_reject_from_mask(fit, mask);
      cpl_mask_delete(mask);
    }
    /* fill everything but the fitted area with ones, so that we  *
     * don't add rubbish when now dividing by this vignetting fix */
    cpl_image_fill_rejected(fit, 1.);

    /* append to reconstructed images */
    image = muse_image_new();
    image->header = cpl_propertylist_new();
    cpl_propertylist_append_string(image->header, "OBJECT", "vignetting fit");
    /* no sensible unit for this image */
    cpl_propertylist_append_string(image->header, "BUNIT", "");
    image->data = fit;
    image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(fit));
    muse_imagelist_set(cubefit->recimages, image,
                       muse_imagelist_get_size(cubefit->recimages));
    cpl_array_set_size(cubefit->recnames,
                       cpl_array_get_size(cubefit->recnames) + 1);
    cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
                         "vign_fit");

    /* divide by the result to assess how well it worked, and append to recimages */
    image = muse_image_new();
    image->header = cpl_propertylist_new();
    cpl_propertylist_append_string(image->header, "OBJECT",
                                   "white image corrected by vignetting fit");
    /* unit again like white image */
    cpl_propertylist_append_string(image->header, "BUNIT", "count");
    image->data = cpl_image_divide_create(wdiv2, fit);
    image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(image->data));
    muse_imagelist_set(cubefit->recimages, image,
                       muse_imagelist_get_size(cubefit->recimages));
    cpl_array_set_size(cubefit->recnames,
                       cpl_array_get_size(cubefit->recnames) + 1);
    cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
                         "white_cor");
    cpl_image_delete(wdiv2);

    /* apply the vignetting fit on top of the twilight cube */
    cpl_msg_info(__func__, "Combining smooth field of view and vignetted region"
                 " in %d planes", npl);
    for (ipl = 0; ipl < npl; ipl++) {
      cpl_image *cimage = cpl_imagelist_get(cubefit->data, ipl);
      cpl_image_accept_all(cimage);
      double mean1 = cpl_image_get_mean(cimage);
      cpl_image_multiply(cimage, fit);
      /* since the image multiplication merges the bad pixel masks, we need *
       * to reset them again before computing the normalization factor */
      cpl_image_accept_all(cimage);
      double mean2 = cpl_image_get_mean(cimage);
      cpl_image_multiply_scalar(cimage, 1. / mean2);
      double mean3 = cpl_image_get_mean(cimage);
      if (fabs(mean3 - 1.) > FLT_EPSILON) {
        cpl_msg_warning(__func__, "normalization failed in plane %d: mean("
                        "plane) = %f -> %f -> %f", ipl + 1, mean1, mean2, mean3);
      } /* if */
    } /* for ipl (all cube image planes) */
    cpl_mask_delete(vignmask);
  } /* if maskimage */
  cpl_mask_delete(fovmask);
  muse_mask_delete(maskimage);

  rc = muse_processing_save_cube(aProcessing, -1, cubefit,
                                 MUSE_TAG_TWILIGHT_CUBE,
                                 MUSE_CUBE_TYPE_FITS);
  muse_datacube_delete(cubefit);

  return rc == CPL_ERROR_NONE ? 0 : -1;
} /* muse_twilight_compute() */
