/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2019 European Southern Observatory
 *           (C) 2001-2008 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#if HAVE_DRAND48
#define _XOPEN_SOURCE /* force drand48 definition from stdlib */
#endif
#include <cpl.h>
#include <cpl_multiframe.h>
#include <float.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h> /* strncasecmp() */
#ifdef HAVE_GETTIMEOFDAY
#include <sys/time.h> /* gettimeofday */
#endif

#include "muse_utils.h"
#include "muse_instrument.h"

#include "muse_cplwrappers.h"
#include "muse_flux.h"
#include "muse_pfits.h"
#include "muse_resampling.h"
#include "muse_tracing.h"
#include "muse_wcs.h"

/*----------------------------------------------------------------------------*
 *                             Debugging Macros                               *
 *----------------------------------------------------------------------------*/
#define MOFFAT_USE_MUSE_OPTIMIZE 0 /* if non-zero, use the muse_cpl_optimize *
                                    * interface to fit the moffat function   */
#if MOFFAT_USE_MUSE_OPTIMIZE
#include "muse_optimize.h"
#endif

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_utils        Utilities
 *
 * This group handles several utility functions.
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/*----------------------------------------------------------------------------*/
/**
  @brief    Get the pipeline copyright and license
  @return   The copyright and license string

  The function wraps the cpl_get_license() macro and hence returns a pointer to
  the statically allocated license string.  This string should not be modified
  using the returned pointer.
 */
/*----------------------------------------------------------------------------*/
const char *
muse_get_license(void)
{
  return cpl_get_license(PACKAGE_NAME, "2005, 2019");
} /* muse_get_license() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Find out the IFU/channel from which this header originated.
  @param    aHeaders   property list/headers to read from
  @return   the requested value or 0 on error

  This is a wrapper around muse_pfits_has_ifu().
 */
/*----------------------------------------------------------------------------*/
unsigned char
muse_utils_get_ifu(const cpl_propertylist *aHeaders)
{
  unsigned char n;
  for (n = 1; n <= kMuseNumIFUs; n++) {
    if (muse_pfits_has_ifu(aHeaders, n)) {
      return n;
    } /* if */
  } /* for n (all IFU numbers) */
  return 0;
} /* muse_utils_get_ifu() */

/*---------------------------------------------------------------------------*/
/**
  @brief  Return extension number that corresponds to this IFU/channel number.
  @param  aFilename   name of the FITS file
  @param  aIFU        the IFU/channel number
  @return Corresponding extension number, or -1 if not found.
 */
/*---------------------------------------------------------------------------*/
int
muse_utils_get_extension_for_ifu(const char *aFilename, unsigned char aIFU)
{
  cpl_errorstate prestate = cpl_errorstate_get();
  int i, next = cpl_fits_count_extensions(aFilename);
  for (i = 0; i <= next; i++) {
    cpl_propertylist *properties = cpl_propertylist_load(aFilename, i);
    if (muse_pfits_has_ifu(properties, aIFU)) {
      cpl_propertylist_delete(properties);
      return i;
    }
    cpl_propertylist_delete(properties);
  } /* for i (primary header and extensions) */
  cpl_errorstate_set(prestate);
  return -1;
} /* muse_utils_get_extension_for_ifu() */

/*---------------------------------------------------------------------------*/
/**
  @brief  return frameset containing data from an IFU/channel with a certain tag
  @param  aFrames   the input frameset
  @param  aTag      the required DFS tag of the frame (optional)
  @param  aIFU      the IFU/channel number to check for (optional)
  @param  aInvert   if true, invert the selection
  @return the resulting frameset or NULL on failure

  Returns the data in form of a frameset. Note that this frameset may be empty
  (but non-NULL), if no frame matching the parameters was found! If aTag is
  NULL, tags are not checked. If aIFU is 0, the IFU number is not checked.

  If aTag is MUSE_TAG_GEOMETRY_TABLE or MUSE_TAG_TWILIGHT_CUBE, then all frames
  with that tag are matched, irrespective of the IFU number found it the header
  of the corresponding file.

  The IFU number is checked using muse_utils_get_ifu().  If this fails, then the
  respective file is expected to be a generic calibration file.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aFrames is NULL}
  @error{propagate error code\, continue with next frame,
         header of a file cannot be loaded}
 */
/*---------------------------------------------------------------------------*/
cpl_frameset *
muse_frameset_find(const cpl_frameset *aFrames, const char *aTag,
                   unsigned char aIFU, cpl_boolean aInvert)
{
  cpl_ensure(aFrames, CPL_ERROR_NULL_INPUT, NULL);
  cpl_frameset *newFrames = cpl_frameset_new();

  /* loop through all frames to determine properties */
  cpl_size iframe, nframes = cpl_frameset_get_size(aFrames);
  for (iframe = 0; iframe < nframes; iframe++) {
    const cpl_frame *frame = cpl_frameset_get_position_const(aFrames, iframe);
    const char *fn = cpl_frame_get_filename(frame),
               *tag = cpl_frame_get_tag(frame);

    /* we are happy, if we want matched frames and found them, or we *
     * want non-matching frames and found one (or one without a tag) */
    cpl_boolean matched = !aInvert && (!aTag || (aTag && !strcmp(tag, aTag))),
                unmatched = aInvert
                          && ((aTag && !tag) || (aTag && strcmp(tag, aTag)));
    if (matched || unmatched) {
      /* files produced outside the pipeline are generic, *
       * so find out if this is one                       */
      cpl_errorstate prestate = cpl_errorstate_get();
      int extension = muse_utils_get_extension_for_ifu(fn, aIFU);
      /* if we haven't found an extension, we should *
       * still try to load the primary header        */
      if (extension == -1) {
        extension = 0;
        cpl_errorstate_set(prestate);
      }
      cpl_propertylist *header = cpl_propertylist_load(fn, extension);
      if (!header) {
        /* ignore files where the header cannot be loaded, but do not *
         * swallow the error state that this failure has already set  */
        continue;
      }
      unsigned char ifu = muse_utils_get_ifu(header);
      prestate = cpl_errorstate_get();
      const char *pipefile = muse_pfits_get_pipefile(header);
      if (!cpl_errorstate_is_equal(prestate)) {
        /* recover from missing PIPEFILE which does not exist in raw data */
        cpl_errorstate_set(prestate);
      }
      if (ifu == aIFU) {
        /* a pipeline produced file which contains data from the correct IFU */
        cpl_frameset_insert(newFrames, cpl_frame_duplicate(frame));
      } else if (ifu == 0 && !pipefile) {
        /* not specific to any IFU, this is the case for external *
         * calibration files, like EXTINCT_TABLE or LINE_CATALOG  */
        cpl_frameset_insert(newFrames, cpl_frame_duplicate(frame));
      } else if (aIFU == 0) {
        /* no IFU specified, this happens in the post-processing recipes */
        cpl_frameset_insert(newFrames, cpl_frame_duplicate(frame));
      } else if (aTag && (!strncmp(aTag, MUSE_TAG_GEOMETRY_TABLE,
                                   strlen(MUSE_TAG_GEOMETRY_TABLE) + 1) ||
                          !strncmp(aTag, MUSE_TAG_TWILIGHT_CUBE,
                                   strlen(MUSE_TAG_TWILIGHT_CUBE) + 1))) {
        /* since the geometry table and twilight cube are not IFU-specific, *
         * they should always match, even if a specific IFU is requested    */
        cpl_frameset_insert(newFrames, cpl_frame_duplicate(frame));
      } else {
        /* do nothing */
#if 0
        cpl_msg_debug(__func__, "not added %s / %s / %d", fn,
                      cpl_frame_get_tag(frame), ifu);
#endif
      }
      cpl_propertylist_delete(header);
    } /* if */
  } /* for iframe (all frames) */

  return newFrames;
} /* muse_frameset_find() */

/*---------------------------------------------------------------------------*/
/**
  @brief  return frameset containing data from an IFU/channel with the given
          tag(s)
  @param  aFrames   the input frameset
  @param  aTags     the required DFS tag(s) of the frame
  @param  aIFU      the IFU/channel number to check for (optional)
  @param  aInvert   if true, invert the selection
  @return the resulting frameset or NULL on failure

  Returns the data in form of a frameset. Note that this frameset may be empty
  (but non-NULL), if no frame matching the parameters was found! If aIFU is 0,
  the IFU number is not checked.

  This function just loops over @ref muse_frameset_find() for all tags in aTags.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aFrames is NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_frameset *
muse_frameset_find_tags(const cpl_frameset *aFrames, const cpl_array *aTags,
                        unsigned char aIFU, cpl_boolean aInvert)
{
  cpl_ensure(aFrames && aTags, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(cpl_array_get_type(aTags) == CPL_TYPE_STRING,
             CPL_ERROR_ILLEGAL_INPUT, NULL);

  cpl_frameset *outframes = cpl_frameset_new();
  cpl_size itag, ntags = cpl_array_get_size(aTags);
  for (itag = 0; itag < ntags; itag++) {
    const char *tag = cpl_array_get_string(aTags, itag);
    cpl_frameset *frames = muse_frameset_find(aFrames, tag, aIFU, aInvert);
    cpl_frameset_join(outframes, frames);
    cpl_frameset_delete(frames);
  } /* for itag */
  return outframes;
} /* muse_frameset_find_tags() */

/*---------------------------------------------------------------------------*/
/**
  @brief  return frameset containing good raw input data
  @param  aFrames   the input frameset
  @param  aTags     the required DFS tag(s) of the frame (optional)
  @param  aIFU      the IFU/channel number
  @return the resulting frameset or NULL on failure

  Returns the good raw input data in form of a frameset. If aTag is
  NULL, tags are not checked.
 */
/*---------------------------------------------------------------------------*/
cpl_frameset *
muse_frameset_check_raw(const cpl_frameset *aFrames, const cpl_array *aTags,
                        unsigned char aIFU)
{
  cpl_frameset *foundFrames = muse_frameset_find_tags(aFrames, aTags, aIFU,
                                                      CPL_FALSE);
  cpl_frameset *newFrames = cpl_frameset_new();

  /* loop through all frames to determine properties */
  cpl_size iframe, nframes = cpl_frameset_get_size(foundFrames);
  cpl_msg_debug(__func__, "Determine properties of all %"CPL_SIZE_FORMAT
                " raw frames of IFU %hhu", nframes, aIFU);
  int binx = -1, biny = -1, readid = -1;
  char *fn = NULL, *readname = NULL, *chipname = NULL, *chipid = NULL;
  for (iframe = 0; iframe < nframes; iframe++) {
    const cpl_frame *frame = cpl_frameset_get_position_const(foundFrames, iframe);
    /* load primary and append IFU extension header (extension is *
     * necessary to get the binning keywords related to raw data) */
    const char *fn2 = cpl_frame_get_filename(frame);
    if (!fn) {
      fn = cpl_strdup(fn2);
    }
    cpl_propertylist *header = cpl_propertylist_load(fn2, 0);
    if (!header) {
      cpl_msg_warning(__func__, "Cannot read primary FITS header of file "
                      "\"%s\"!", fn2);
      continue;
    }
    int extension = muse_utils_get_extension_for_ifu(fn2, aIFU);
    if (extension > 0) {
      cpl_propertylist *exthead = cpl_propertylist_load(fn2, extension);
      cpl_propertylist_append(header, exthead);
      cpl_propertylist_delete(exthead);
    }
    cpl_boolean isOK = CPL_TRUE;
    if (binx < 0) {
      binx = muse_pfits_get_binx(header);
    }
    if (biny < 0) {
      biny = muse_pfits_get_biny(header);
    }
    if (!readname) {
      readname = cpl_strdup(muse_pfits_get_read_name(header));
    }
    if (readid < 0) {
      readid = muse_pfits_get_read_id(header);
    }
    if (!chipname) {
      chipname = cpl_strdup(muse_pfits_get_chip_name(header));
    }
    if (!chipid) {
      chipid = cpl_strdup(muse_pfits_get_chip_id(header));
    }
    int binx2 = muse_pfits_get_binx(header),
        biny2 = muse_pfits_get_biny(header),
        readid2 = muse_pfits_get_read_id(header);
    const char *chipname2 = muse_pfits_get_chip_name(header),
               *chipid2 = muse_pfits_get_chip_id(header);
    if (binx2 != binx) {
      cpl_msg_warning(__func__, "File \"%s\" (IFU %hhu) was taken with a different"
                      " x-binning factor (reference \"%s\", %d instead of %d)!",
                      fn2, aIFU, fn, binx2, binx);
      isOK = 0;
    }
    if (biny2 != biny) {
      cpl_msg_warning(__func__, "File \"%s\" (IFU %hhu) was taken with a different"
                      " y-binning factor (reference \"%s\", %d instead of %d)!",
                      fn2, aIFU, fn, biny2, biny);
      isOK = 0;
    }
    if (readid2 != readid) {
      cpl_msg_warning(__func__, "File \"%s\" (IFU %hhu) was taken with a different"
                      " read-out mode (reference \"%s\", %d/%s instead of %d/%s)!",
                      fn2, aIFU, fn, readid2, muse_pfits_get_read_name(header),
                      readid, readname);
      isOK = 0;
    }
    if (!chipname2 || !chipid2 ||
        strcmp(chipname, chipname2) || strcmp(chipid, chipid2)) {
      cpl_msg_warning(__func__, "File \"%s\" (IFU %hhu) has a different chip "
                      "setup (reference \"%s\", name %s vs %s, id %s vs %s)",
                      fn2, aIFU, fn, chipname2, chipname, chipid2, chipid);
      isOK = 0;
    }

    if (!cpl_frame_get_tag(frame) ||
        !strncmp(cpl_frame_get_tag(frame), MUSE_TAG_EMPTY, 1) ) {
      /* non-fatal, continue normally and check for other possible *
       * problems, that should override this exclude reason        */
      cpl_msg_warning(__func__, "File \"%s\" (IFU %hhu) is not tagged!", fn2,
                      aIFU);
    }
    cpl_propertylist_delete(header);
    if (isOK) {
      cpl_frameset_insert(newFrames, cpl_frame_duplicate(frame));
    }
  } /* for nframes */
  cpl_free(fn);
  cpl_free(readname);
  cpl_free(chipname);
  cpl_free(chipid);
  cpl_frameset_delete(foundFrames);

  return newFrames;
} /* muse_frameset_check_raw() */

/*---------------------------------------------------------------------------*/
/**
  @brief  Create a new frameset containing all relevant raw frames first then
          all other frames.
  @param  aFrames     the frames list
  @param  aIndex      the frame index (use a negative value to signify to take
                      all raw frames or use aDateObs for comparison)
  @param  aDateObs    the DATE-OBS string to search for (only when aIndex is
                      negative, use NULL to signify to take all frames)
  @param  aSequence   if target frame is part of a sequence
  @return The new ordered frameset or NULL on error.

  Normal raw frames are sorted first, if their location in aFrames match the
  aIndex, or aDateObs is given and the DATE-OBS in their header match it. If
  aSequence is true, then it is expected that they are part of an exposure
  sequence (e.g. calibration flats), and they are taken irrespective of aIndex.

  Raw frames tagged "ILLUM" also get a special handling, in that the first of
  them -- i.e. the one that got used in muse_basicproc_get_illum() -- is sorted
  into the output frameset after all other raw frames, but before all other
  frames.

  The returned frameset is duplicated from the input frameset and therefore has
  to be deallocated using cpl_frameset_delete() after use.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, input frameset was NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_frameset *
muse_frameset_sort_raw_other(const cpl_frameset *aFrames, int aIndex,
                             const char *aDateObs, cpl_boolean aSequence)
{
  cpl_ensure(aFrames, CPL_ERROR_NULL_INPUT, NULL);

  cpl_frameset *fraw = cpl_frameset_new(),
               *fillum = cpl_frameset_new(),
               *fother = cpl_frameset_new();
  int iraw = 0, /* raw frame index */
      nillum = 0; /* counter for ILLUM frames */
  cpl_size iframe, nframes = cpl_frameset_get_size(aFrames);
  for (iframe = 0; iframe < nframes; iframe++) {
    const cpl_frame *frame = cpl_frameset_get_position_const(aFrames, iframe);
    if (cpl_frame_get_group(frame) == CPL_FRAME_GROUP_RAW) {
      const char *tag = cpl_frame_get_tag(frame);
      if (!tag || (tag && strncmp(tag, "ILLUM", 6))) {
        /* "normal" raw frame:                               *
         * if the index matches, insert it, otherwise ignore */
        int datematch = 1; /* pretend to match when aDateObs == NULL */
        if (aDateObs) {
          cpl_propertylist *header
            = cpl_propertylist_load(cpl_frame_get_filename(frame), 0);
          const char *dateobs = muse_pfits_get_dateobs(header);
          datematch = dateobs && !strncmp(aDateObs, dateobs, strlen(aDateObs));
          cpl_propertylist_delete(header);
        }
        if ((aIndex < 0 && datematch) || aIndex == iraw || aSequence) {
          cpl_frameset_insert(fraw, cpl_frame_duplicate(frame));
        }
        iraw++;
      } else {
        /* raw ILLUM:                                         *
         * if it's the first one, insert it, otherwise ignore */
        if (!nillum) {
          cpl_frameset_insert(fillum, cpl_frame_duplicate(frame));
        }
        nillum++;
      } /* else: is ILLUM */
    } else {
      cpl_frameset_insert(fother, cpl_frame_duplicate(frame));
    } /* else: not GROUP_RAW */
  } /* for all incoming frames */
#if 0
  printf("incoming frames (index=%d, DATE-OBS=%s):\n", aIndex, aDateObs);
  cpl_frameset_dump(aFrames, stdout);
  fflush(stdout);
  printf("=============================================================\n"
         "raw frames:\n");
  cpl_frameset_dump(fraw, stdout);
  printf("illum frames:\n");
  cpl_frameset_dump(fillum, stdout);
  printf("-------------------------------------------------------------\n"
         "other frames:\n");
  cpl_frameset_dump(fother, stdout);
  printf("^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^\n");
  fflush(stdout);
#endif

  /* transfer all other frames into the raw = output frameset */
  cpl_frameset_join(fraw, fillum);
  cpl_frameset_join(fraw, fother);
  cpl_frameset_delete(fother);
  cpl_frameset_delete(fillum);
#if 0
  printf("=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=_=\n"
         "all sorted frames:\n");
  cpl_frameset_dump(fraw, stdout);
  printf("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n");
  fflush(stdout);
#endif

  return fraw;
} /* muse_frameset_sort_raw_other() */

/*---------------------------------------------------------------------------*/
/**
  @brief  find the master frame according to its CCD number and tag
  @param  aFrames   the frames list
  @param  aTag      the tag
  @param  aIFU      the IFU/channel number
  @return the frame of the master image

  The returned frame is duplicated from the input frameset and therefore has
  to be deallocated after use.
 */
/*---------------------------------------------------------------------------*/
cpl_frame *
muse_frameset_find_master(const cpl_frameset *aFrames, const char *aTag,
                          unsigned char aIFU)
{
  cpl_frameset *frames = muse_frameset_find(aFrames, aTag, aIFU, CPL_FALSE);
  cpl_frame *frame = NULL;
  if (cpl_frameset_count_tags(frames, aTag) == 1) {
    frame = cpl_frame_duplicate(cpl_frameset_get_position_const(frames, 0));
  }
  cpl_frameset_delete(frames);
  return frame;
} /* muse_frameset_find_master() */

/* Remove .fits and possibly -nn from a filename of the style  *
 * "SOME_TAG_iiii-nn.fits".                                    *
 * The returned string has to be deallocated using cpl_free(). */
static char *
muse_utils_frame_get_basefilename(const cpl_frame *aFrame)
{
  char *filename = cpl_strdup(cpl_frame_get_filename(aFrame)),
       *end = strstr(filename, ".fits");
  if (end) {
    *end = '\0'; /* strip off ".fits" */
  }
  end = strrchr(filename, '-');
  if (end) {
    *end = '\0'; /* strip off the "-nn" extension number */
  }
  return filename;
} /* muse_utils_frame_get_basefilename() */

/* Compare the base filenames of two frames. *
 * To be used with cpl_frameset_labelise().  */
static int
muse_utils_frames_compare_basenames(const cpl_frame *aF1, const cpl_frame *aF2)
{
  cpl_ensure(aF1 && aF2, CPL_ERROR_NULL_INPUT, -1);
  cpl_ensure(cpl_frame_get_filename(aF1) && cpl_frame_get_filename(aF2),
             CPL_ERROR_DATA_NOT_FOUND, -1);
  char *fn1 = muse_utils_frame_get_basefilename(aF1),
       *fn2 = muse_utils_frame_get_basefilename(aF2);
  int cmp = strcmp(fn1, fn2);
  cpl_free(fn1);
  cpl_free(fn2);
  return cmp == 0 ? 1 : 0;
} /* muse_utils_frames_compare_basenames() */

/* Compare the full filenames of two frames. *
 * To be used with cpl_frameset_sort().      */
static int
muse_utils_frame_compare(const cpl_frame *aF1, const cpl_frame *aF2)
{
  const char *fn1 = cpl_frame_get_filename(aF1),
             *fn2 = cpl_frame_get_filename(aF2);
  int cmp = strcmp(fn1, fn2);
  return (cmp < 0) ? -1 : (cmp > 0) ? 1 : 0;
}

/*---------------------------------------------------------------------------*/
/**
  @brief  Merge IFU-specific files from a frameset to create multi-IFU outputs.
  @param  aFrames   the frames list
  @param  aDelete   if CPL_TRUE, delete input files from framesete after merging
  @return CPL_ERROR_NONE on success another CPL error code on failure

  This function groups all frames in the input frameset regarding the base of
  the filenames (without -nn.fits), and then merges them, taking care to put
  IFU-specific information into the extension header (like QC and DRS
  parameters).
  The output filename for each group is the common basename of files (usually
  the PRO.CATG tag given to it by the pipeline during processing).

  After all products were merged, the related files in the input frameset are
  physically removed from the filesystem and purged from aFrames.

  The merged files in the frameset aFrames are set to the cpl_frame_group of
  CPL_FRAME_GROUP_PRODUCT, even if the input frames were not.

  @note Files tagged with "PIXTABLE_*" are not merged, but silently ignored!

  @error{set and return CPL_ERROR_NULL_INPUT, aFrames is NULL}
  @error{set and return CPL_ERROR_ILLEGAL_INPUT, aFrames is empty}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_utils_frameset_merge_frames(cpl_frameset *aFrames, cpl_boolean aDelete)
{
  cpl_ensure_code(aFrames, CPL_ERROR_NULL_INPUT);
  /* better fail here on empty frameset than in cpl_frameset_labelise() */
  cpl_ensure_code(cpl_frameset_get_size(aFrames) > 0, CPL_ERROR_ILLEGAL_INPUT);
#if 0
  printf("final out frames:\n");
  cpl_frameset_dump(aFrames, stdout);
  fflush(stdout);
#endif

  /* set up keys and regular expression (positive and negative ones!) *
   * for the merged extensions                                        */
#define EXTKEYS1 MUSE_WCS_KEYS"|(ESO DET (CHIP|OUT[1-9]*) |ESO QC|ESO DRS)"
#define EXTKEYS2 MUSE_WCS_KEYS"|^B(UNIT|SCALE|ZERO)"
  cpl_regex *regex = cpl_regex_new(EXTKEYS1, TRUE, CPL_REGEX_EXTENDED),
            *nregex = cpl_regex_new(EXTKEYS1, FALSE, CPL_REGEX_EXTENDED),
            *nregex2 = cpl_regex_new(EXTKEYS1"|"EXTKEYS2, FALSE,
                                     CPL_REGEX_EXTENDED);

  /* check sets of output frames */
  cpl_frameset *frames = cpl_frameset_new();

  cpl_size ilabel;
  cpl_size nlabels = 0;
  cpl_size *labels = cpl_frameset_labelise(aFrames,
                                           muse_utils_frames_compare_basenames,
                                           &nlabels);
  for (ilabel = 0; ilabel < nlabels; ilabel++) {
    cpl_frameset *fset = cpl_frameset_extract(aFrames, labels, ilabel);
    /* sort the frameset to get a nice ordering in the merged output, *
     * and to get the primary header propagated from the first IFU    */
    cpl_frameset_sort(fset, muse_utils_frame_compare);

    cpl_frame *frame = cpl_frameset_get_position(fset, 0);
    const char *tag = cpl_frame_get_tag(frame);
    /* XXX This is a workaround to exclude pixel tables from being  *
     *     merged, since merged pixel tables can not yet be used by *
     *     the post-processing recipes.                             */
    if (strncmp(tag, "PIXTABLE_", 9) == 0) {
      cpl_frameset_delete(fset);
      continue;
    }

    int n = cpl_frameset_get_size(fset);
    if (n <= 1) {
      cpl_msg_warning(__func__, "Nothing to merge for tag %s (%d frames)!",
                      tag, n);
      cpl_frameset_delete(fset);
      continue;
    }

    cpl_multiframe *mf = cpl_multiframe_new(frame, "", regex);
    if (!mf) {
      cpl_frameset_delete(fset);
      continue;
    }

    /* now loop through all frames with this tag and append them to the group */
    int i;
    for (i = 0; i < n; i++) {
      frame = cpl_frameset_get_position(fset, i);
      const char *fn = cpl_frame_get_filename(frame);
      cpl_msg_debug(__func__, "Merging \"%s\".", fn);
      /* check, if this is a three-extension muse_image or something else */
      int extdata = cpl_fits_find_extension(fn, EXTNAME_DATA),
          extdq = cpl_fits_find_extension(fn, EXTNAME_DQ),
          extstat = cpl_fits_find_extension(fn, EXTNAME_STAT);
      cpl_errorstate state = cpl_errorstate_get();
      if (extdata > 0 && extdq > 0 && extstat > 0) {
        /* is a three-extension muse_image */
        const char *extnames[] = { EXTNAME_DATA, EXTNAME_DQ, EXTNAME_STAT },
                   *updkeys[] = { "SCIDATA", "ERRDATA", "QUALDATA", NULL };
        const cpl_regex *filters[] = { nregex, nregex, nregex };
        cpl_multiframe_append_datagroup(mf, ".", frame, 3, extnames, filters,
                                        NULL, updkeys, CPL_MULTIFRAME_ID_JOIN);
      } else if (cpl_fits_count_extensions(fn) == 0) {
        /* is an image in the primary HDU                                   *
         * needs a different set of keywords to transfer the relevant stuff */
        cpl_multiframe_append_dataset_from_position(mf, ".", frame, 0,
                                                    nregex2, NULL,
                                                    CPL_MULTIFRAME_ID_JOIN);
      } else {
        /* is a table or multi-extension file */
        int iext, next = cpl_fits_count_extensions(fn);
        for (iext = 1; iext <= next; iext++) {
          cpl_multiframe_append_dataset_from_position(mf, ".", frame, iext,
                                                      nregex, NULL,
                                                      CPL_MULTIFRAME_ID_JOIN);
        } /* for iext (all extensions) */
      } /* else */

      if (!cpl_errorstate_is_equal(state)) {
        cpl_msg_error(__func__, "Appending data of \"%s\" for merging failed: %s",
                      fn, cpl_error_get_message());
      } /* if error */
    } /* for i (all frames with this tag) */

    /* now finally write the merged file */
    char *basefn = muse_utils_frame_get_basefilename(frame),
         *outfn = cpl_sprintf("%s.fits", basefn);
    cpl_free(basefn);
    cpl_errorstate state = cpl_errorstate_get();
    cpl_multiframe_write(mf, outfn);
    if (!cpl_errorstate_is_equal(state)) {
      cpl_msg_error(__func__, "Writing merged data to \"%s\" failed: %s",
                    outfn, cpl_error_get_message());
    } else {
      cpl_frame_set_filename(frame, outfn);
      /* set the frame's group to product, to easily *
       * set them apart from other input frames      */
      cpl_frame_set_group(frame, CPL_FRAME_GROUP_PRODUCT);
      /* record the merged file in the resulting frameset */
      cpl_frameset_insert(frames, cpl_frame_duplicate(frame));
    }
    cpl_free(outfn);
    cpl_multiframe_delete(mf);
    cpl_frameset_delete(fset);
  } /* for ilabel (all frame labels) */
  cpl_regex_delete(regex);
  cpl_regex_delete(nregex);
  cpl_regex_delete(nregex2);
  cpl_free(labels);
#if 0
  printf("\naFrames (trying to remove these):\n");
  cpl_frameset_dump(aFrames, stdout);
  fflush(stdout);
  printf("\nmerged frames:\n");
  cpl_frameset_dump(frames, stdout);
  fflush(stdout);
#endif

  /* remove all frames for the successfully merged tags */
  int iframe, nframes = cpl_frameset_get_size(frames);
  for (iframe = 0; aDelete == CPL_TRUE && iframe < nframes; iframe++) {
    cpl_frame *frame = cpl_frameset_get_position(frames, iframe);
#if 1
    cpl_msg_debug(__func__, "===== Starting to compare \"%s\" =====",
                  cpl_frame_get_filename(frame));
#endif
    int iframe2;
    for (iframe2 = 0; iframe2 < cpl_frameset_get_size(aFrames); iframe2++) {
      /* duplicate, so that we can use it to remove "itself" */
      cpl_frame *frame2 = cpl_frameset_get_position(aFrames, iframe2);
      if (muse_utils_frames_compare_basenames(frame, frame2) == 1) {
        const char *fn = cpl_frame_get_filename(frame2);
#if 1
        char *fb1 = muse_utils_frame_get_basefilename(frame),
             *fb2 = muse_utils_frame_get_basefilename(frame2);
        cpl_msg_debug(__func__, "Removing \"%s\" (\"%s\" vs \"%s\").",
                      fn, fb1, fb2);
        cpl_free(fb1);
        cpl_free(fb2);
#endif
        remove(fn);
        cpl_frameset_erase_frame(aFrames, frame2);
        iframe2--; /* stay here to see what moved here */
      } /* if */
    } /* for iframe2 */
  } /* for iframe (all frames in new frameset) */
  cpl_frameset_join(aFrames, frames);
  cpl_frameset_delete(frames);
#if 0
  printf("\nmerged out frames:\n");
  cpl_frameset_dump(aFrames, stdout);
  fflush(stdout);
#endif
  return CPL_ERROR_NONE;
} /* muse_utils_frameset_merge_frames() */

/*----------------------------------------------------------------------------*/
/**
 * @brief MUSE filter table definition.
 *
 * A MUSE filter table has the following columns:
 * - 'lambda': the wavelength in Angstrom
 * - 'throughput': the filter response in fractions of 1
 */
/*----------------------------------------------------------------------------*/
const muse_cpltable_def muse_filtertable_def[] = {
  { "lambda", CPL_TYPE_DOUBLE, "Angstrom", "%7.2f", "wavelength", CPL_TRUE},
  { "throughput", CPL_TYPE_DOUBLE, "", "%.4e",
    "filter response (in fractions of 1)", CPL_TRUE},
  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
};

/*---------------------------------------------------------------------------*/
/**
  @brief  Load a table for a given filter name.
  @param  aProcessing   the processing structure, includes the input frames list
  @param  aFilterName   the filter name
  @return the cpl_table * or NULL on error

  This function tries to find a filter table and in it an extension for the
  filter of the given name. The table from that extension is loaded.

  A filter called "white" is built into this function, it covers the full MUSE
  wavelength range with constant throughput.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aFilterName is NULL}
  @error{return NULL (and output debug message), aFilterName is "none"}
  @error{set CPL_ERROR_FILE_NOT_FOUND\, return NULL,
         input frame with tag MUSE_TAG_FILTER_LIST was not found}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         valid file with tag MUSE_TAG_FILTER_LIST does not contain filter named aFilterName}
  @error{propagate error code\, return NULL,
         loading table for filter aFilterName from valid file with tag MUSE_TAG_FILTER_LIST failed}
 */
/*---------------------------------------------------------------------------*/
muse_table *
muse_table_load_filter(muse_processing *aProcessing, const char *aFilterName)
{
  cpl_ensure(aFilterName, CPL_ERROR_NULL_INPUT, NULL);
  if (!strncasecmp(aFilterName, "none", 4)) {
    cpl_msg_debug(__func__, "No filter wanted (filter \"%s\")", aFilterName);
    return NULL;
  }
  if (!strcmp(aFilterName, "white")) {
    cpl_msg_debug(__func__, "White-light integration (internal filter \"%s\")",
                  aFilterName);
    /* create minimal rectangular filter function table */
    cpl_table *table = muse_cpltable_new(muse_filtertable_def, 4);
    cpl_table_set(table, "lambda", 0, kMuseNominalLambdaMin - 1e-5);
    cpl_table_set(table, "lambda", 1, kMuseNominalLambdaMin);
    cpl_table_set(table, "lambda", 2, kMuseNominalLambdaMax);
    cpl_table_set(table, "lambda", 3, kMuseNominalLambdaMax + 1e-5);
    cpl_table_set(table, "throughput", 0, 0.);
    cpl_table_set(table, "throughput", 1, 1.);
    cpl_table_set(table, "throughput", 2, 1.);
    cpl_table_set(table, "throughput", 3, 0.);
    /* now convert into MUSE table with header */
    muse_table *mt = muse_table_new();
    mt->table = table;
    mt->header = cpl_propertylist_new();
    cpl_propertylist_append_string(mt->header, "EXTNAME", "white");
    return mt;
  } /* if white-light filter */
  cpl_frame *frame = muse_frameset_find_master(aProcessing->inframes,
                                               MUSE_TAG_FILTER_LIST, 0);
  if (!frame) {
    cpl_error_set_message(__func__, CPL_ERROR_FILE_NOT_FOUND, "%s (for filter "
                          "\"%s\") is missing", MUSE_TAG_FILTER_LIST,
                          aFilterName);
    return NULL;
  }

  /* try to load the extension table for the given filter, using EXTNAME */
  char *filename = (char *)cpl_frame_get_filename(frame);
  int ext = cpl_fits_find_extension(filename, aFilterName);
  if (ext <= 0) {
    cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "\"%s\" does not "
                          "contain filter \"%s\"", filename, aFilterName);
    cpl_frame_delete(frame);
    return NULL;
  }

  /* load table, and create header composed of primary plus extension keys */
  muse_table *table = muse_table_new();
  table->header = cpl_propertylist_load(filename, 0); /* primay header */
  if (!table->header) {
    cpl_error_set_message(__func__, cpl_error_get_code(), "loading filter "
                          "\"%s\" from file \"%s\" (ext %d) failed",
                          aFilterName, filename, ext);
    cpl_frame_delete(frame);
    muse_table_delete(table);
    return NULL;
  }
  /* load the table itself */
  table->table = cpl_table_load(filename, ext, 1);
  if (!table->table || !cpl_table_get_nrow(table->table)) {
    cpl_error_set_message(__func__, cpl_error_get_code(), "loading filter "
                          "\"%s\" from file \"%s\" (ext %d) failed",
                          aFilterName, filename, ext);
    cpl_frame_delete(frame);
    muse_table_delete(table);
    return NULL;
  }
  /* merge in important keywords from the extension header */
  cpl_propertylist *hext = cpl_propertylist_load(filename, ext);
  cpl_propertylist_copy_property_regexp(table->header, hext,
                                        "^EXTNAME$|^Z|^COMMENT", 0);
  cpl_propertylist_delete(hext);

  cpl_msg_info(__func__, "loaded filter \"%s\" from file \"%s\" (ext %d)",
               aFilterName, filename, ext);
  /* filters from the input frameset should be listed, append them into       *
   * the used frames for the FITS header of the output product, if they exist */
  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_CALIB, 0);
  return table;
} /* muse_table_load_filter() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Compute fraction of filter area covered by the data range.
  @param   aFilter    the filter response curve
  @param   aLambda1   the starting (data) wavelength
  @param   aLambda2   the end (data) wavelength
  @return  the fraction or -1. on error.

  @error{set CPL_ERROR_NULL_INPUT\, return -1.,
         aFilter or its table component are NULL}
 */
/*---------------------------------------------------------------------------*/
double
muse_utils_filter_fraction(const muse_table *aFilter, double aLambda1,
                           double aLambda2)
{
  cpl_ensure(aFilter && aFilter->table, CPL_ERROR_NULL_INPUT, -1.);

  /* get the extreme wavelengths of the filter function */
  const cpl_table *table = aFilter->table; /* shortcut */
  int nrow = cpl_table_get_nrow(table);
  double lbda1 = cpl_table_get(table, "lambda", 0, NULL),
         lbda2 = cpl_table_get(table, "lambda", nrow - 1, NULL);

  /* go through all wavelengths with constant delta-lambda = 1 *
   * and compute the summed filter area, over the total filter *
   * and over the the wavelength range given (by the data)     */
  double sumt = 0., /* the total sum */
         sum2 = 0., /* the sum over the wavelength range */
         lambda;
  for (lambda = lbda1; lambda <= lbda2; lambda++) {
    double thruput = muse_flux_response_interpolate(table, lambda, NULL,
                                                    MUSE_FLUX_RESP_FILTER);
    sumt += thruput;
    if (lambda >= aLambda1 && lambda <= aLambda2) {
      sum2 += thruput;
    }
  } /* for lambda (all wavelengths in filter) */
  return sum2 / sumt; /* return the fraction */
} /* muse_utils_filter_fraction() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Add/propagate filter properties to header of collapsed image.
  @param   aHeader     the header to add to
  @param   aFilter     the filter response curve
  @param   aFraction   the filter covering fraction
  @return CPL_ERROR_NONE or a CPL error code on failure.

  @error{set and return CPL_ERROR_NULL_INPUT,
         aHeader\, aFilter\, or the header component of aFilter are NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_utils_filter_copy_properties(cpl_propertylist *aHeader,
                                  const muse_table *aFilter, double aFraction)
{
  cpl_ensure_code(aHeader && aFilter && aFilter->header, CPL_ERROR_NULL_INPUT);

  /* if a filter was passed, propagate name and photometric zeropoints *
   * from filter and add coverage fraction                             */
  cpl_propertylist_update_string(aHeader, MUSE_HDR_FILTER,
                                 cpl_propertylist_get_string(aFilter->header,
                                                             "EXTNAME"));
  cpl_propertylist_set_comment(aHeader, MUSE_HDR_FILTER, MUSE_HDR_FILTER_C);
  if (cpl_propertylist_has(aFilter->header, "ZP_VEGA")) {
    cpl_propertylist_update_double(aHeader, MUSE_HDR_FILTER_ZPVEGA,
                                   cpl_propertylist_get_double(aFilter->header,
                                                               "ZP_VEGA"));
    cpl_propertylist_set_comment(aHeader, MUSE_HDR_FILTER_ZPVEGA,
                                 MUSE_HDR_FILTER_ZPVEGA_C);
  }
  if (cpl_propertylist_has(aFilter->header, "ZP_AB")) {
    cpl_propertylist_update_double(aHeader, MUSE_HDR_FILTER_ZPAB,
                                   cpl_propertylist_get_double(aFilter->header,
                                                               "ZP_AB"));
    cpl_propertylist_set_comment(aHeader, MUSE_HDR_FILTER_ZPAB,
                                 MUSE_HDR_FILTER_ZPAB_C);
  }
  cpl_propertylist_update_float(aHeader, MUSE_HDR_FILTER_FFRAC,
                                aFraction * 100.);
  cpl_propertylist_set_comment(aHeader, MUSE_HDR_FILTER_FFRAC,
                               MUSE_HDR_FILTER_FFRAC_C);

  return CPL_ERROR_NONE;
} /* muse_utils_filter_copy_properties() */

/*---------------------------------------------------------------------------*/
/**
  @brief  Concatenate names of all active calibration lamps.
  @param  aHeader   the FITS header to search for lamp entries
  @param  aSep      the separator to use for name concatenation
  @return A newly allocated string with the filter name(s) or NULL on error or
          if no lamps are switched on.

  This function iterates through all lamp shutters found in the header, and for
  those where the lamp is switched on and the shutter is open appends the name
  to the output string.
  Prefixes of lamp names (currently only "CU-LAMP-") are cut off, e.g.
  "CU-LAMP-HgCd" becomes "HgCd".
  XXX to aid AIT, lamp names like "CU-LAMP[3-6]" are converted to the respective
      element names (Ne, Xe, HgCd).

  The returned string has to be deallocated with cpl_free().

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aHeader is NULL}
 */
/*---------------------------------------------------------------------------*/
char *
muse_utils_header_get_lamp_names(cpl_propertylist *aHeader, char aSep)
{
  cpl_ensure(aHeader, CPL_ERROR_NULL_INPUT, NULL);

  char *lampname = NULL;
  int n, nlamps = muse_pfits_get_lampnum(aHeader);
  for (n = 1; n <= nlamps; n++) {
    cpl_errorstate prestate = cpl_errorstate_get();
    int st1 = muse_pfits_get_lamp_status(aHeader, n),
        st2 = muse_pfits_get_shut_status(aHeader, n);
    if (!cpl_errorstate_is_equal(prestate)) {
      cpl_errorstate_set(prestate); /* lamp headers may be missing */
    }
    if (!st1 || !st2) {
      continue; /* lamp off or its shutter closed */
    }
    char *name = /* XXX ! */(char *)muse_pfits_get_lamp_name(aHeader, n);
    if (!strncmp(name, "CU-LAMP-", 8)) {
      name += 8; /* ignore the lamp prefix */
    }
    if (!strcmp(name, "CU-LAMP3") || !strcmp(name, "CU-LAMP6")) { /* XXX overwrite read-only locations */
      /* 3 is standard Neon, 6 is high-power Neon */
      name[0] = 'N';
      name[1] = 'e';
      name[2] = '\0';
    } else if (!strcmp(name, "CU-LAMP4")) {
      name[0] = 'X';
      name[1] = 'e';
      name[2] = '\0';
    } else if (!strcmp(name, "CU-LAMP5")) {
      name[0] = 'H';
      name[1] = 'g';
      name[2] = 'C';
      name[3] = 'd';
      name[4] = '\0';
    }
    if (lampname) {
      char *temp = lampname;
      lampname = cpl_sprintf("%s%c%s", temp, aSep, name);
      cpl_free(temp);
    } else {
      lampname = cpl_sprintf("%s", name);
    }
  } /* for n (all possible lamps) */
  return lampname;
} /* muse_utils_header_get_lamp_names() */

/*---------------------------------------------------------------------------*/
/**
  @brief  List numbers of all active calibration lamps.
  @param  aHeader   the FITS header to search for lamp entries
  @return A cpl_array of type CPL_TYPE_INT, or NULL on error or
          if no lamps are switched on.

  This function iterates through all lamp shutters found in the header, and for
  those where the lamp is switched on and the shutter is open appends the lamp
  number to the output array.

  The lamp numbers are the i in the ESO.INS.LAMPi keywords in the MUSE FITS
  headers.

  The returned array has to be deallocated with cpl_array_delete().

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aHeader is NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_array *
muse_utils_header_get_lamp_numbers(cpl_propertylist *aHeader)
{
  cpl_ensure(aHeader, CPL_ERROR_NULL_INPUT, NULL);

  /* start with an array of zero length */
  cpl_array *lampnumbers = cpl_array_new(0, CPL_TYPE_INT);
  int n, nlamps = muse_pfits_get_lampnum(aHeader);
  for (n = 1; n <= nlamps; n++) {
    cpl_errorstate prestate = cpl_errorstate_get();
    int st1 = muse_pfits_get_lamp_status(aHeader, n),
        st2 = muse_pfits_get_shut_status(aHeader, n);
    if (!cpl_errorstate_is_equal(prestate)) {
      cpl_errorstate_set(prestate); /* lamp headers may be missing */
    }
    if (!st1 || !st2) {
      continue; /* lamp off or its shutter closed */
    }
    /* now enlarge the array and save the lamp numbers as last element */
    cpl_array_set_size(lampnumbers, cpl_array_get_size(lampnumbers) + 1);
    cpl_array_set_int(lampnumbers,  cpl_array_get_size(lampnumbers) - 1,
                      n);
  } /* for n (all possible lamps) */
  if (cpl_array_get_size(lampnumbers) < 1) {
    cpl_array_delete(lampnumbers);
    lampnumbers = NULL;
  }
  return lampnumbers;
} /* muse_utils_header_get_lamp_numbers() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Create a matrix that contains a normalized 2D Gaussian
  @param    aXHalfwidth   horizontal half width of the kernel matrix
  @param    aYHalfwidth   vertical half width of the kernel matrix
  @param    aSigma        sigma of Gaussian function
  @return   The new matrix or NULL on error

  The 2*halfwidth+1 gives the size of one side of the matrix, i.e.
  halfwidth=3 for a 7x7 matrix.
  The created matrix has to be deallocated by cpl_matrix_delete().
 */
/*----------------------------------------------------------------------------*/
cpl_matrix *
muse_matrix_new_gaussian_2d(int aXHalfwidth, int aYHalfwidth, double aSigma)
{
  cpl_matrix *kernel = cpl_matrix_new(2*aXHalfwidth+1, 2*aYHalfwidth+1);
  if (!kernel) {
    cpl_msg_error(__func__, "Could not create matrix: %s",
                  cpl_error_get_message());
    return NULL;
  }
  double sum = 0.;
  int i;
  for (i = -aXHalfwidth; i <= aXHalfwidth; i++) {
    int j;
    for (j = -aYHalfwidth; j <= aYHalfwidth; j++) {
      /* set Gaussian kernel */
      double gauss = 1. / (aSigma*sqrt(2.*CPL_MATH_PI))
                   * exp(-(i*i + j*j) / (2.*aSigma*aSigma));
      cpl_matrix_set(kernel, i+aXHalfwidth, j+aYHalfwidth, gauss);
      sum += gauss;
    }
  }
  /* normalize the matrix, the sum of the elements should be 1 */
  cpl_matrix_divide_scalar(kernel, sum);

  return kernel;
} /* muse_matrix_new_gaussian_2d */

/*----------------------------------------------------------------------------*/
/**
  @brief    Create a smooth version of a 2D image by fitting it with a 2D
            polynomial.
  @param    aImage    input image to fit
  @param    aXOrder   polynomial order to use in x direction
  @param    aYOrder   polynomial order to use in y direction
  @return   a new cpl_image * constructed from the polynomial fit
            or NULL on error

  This function transfers all pixels (their positions and values) into suitable
  structures and calls cpl_polynomial_fit(). If that polynomial fit succeeds,
  cpl_image_fill_polynomial() is used to create the output image. Rejected (bad)
  pixels in the input image are marked as rejected in the output image as well.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aImage is NULL}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         aImage does not contain good pixels}
  @error{propagate error code\, return NULL, cpl_polynomial_fit() fails}
 */
/*----------------------------------------------------------------------------*/
cpl_image *
muse_utils_image_fit_polynomial(const cpl_image *aImage, unsigned short aXOrder,
                                unsigned short aYOrder)
{
  cpl_ensure(aImage, CPL_ERROR_NULL_INPUT, NULL);

  /* transfer all pixel positions and their values *
   * into matrix and vector for the polynomial fit */
  int nx = cpl_image_get_size_x(aImage),
      ny = cpl_image_get_size_y(aImage);
  cpl_matrix *pos = cpl_matrix_new(2, nx * ny);
  cpl_vector *val = cpl_vector_new(nx * ny);
  int i, j, np = 0;
  for (i = 1; i < nx; i++) {
    for (j = 1; j < ny; j++) {
      if (cpl_image_is_rejected(aImage, i, j)) {
        continue;
      }
      cpl_matrix_set(pos, 0, np, i);
      cpl_matrix_set(pos, 1, np, j);
      int err;
      cpl_vector_set(val, np, cpl_image_get(aImage, i, j, &err));
      np++;
    } /* for j (vertical pixels) */
  } /* for i (horizontal pixels) */
  /* vectors and matrices cannot be resized to have zero elements, exit early */
  if (!np) {
    cpl_matrix_delete(pos);
    cpl_vector_delete(val);
    cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND, "No good pixels "
                          "found in image, polynomial fit cannot be performed!");
    return NULL;
  }
  /* re-set sizes depending on the number of valid pixels found */
  cpl_matrix_set_size(pos, 2, np);
  cpl_vector_set_size(val, np);

  /* carry out the fit */
  cpl_polynomial *poly = cpl_polynomial_new(2);
  const cpl_boolean sym = CPL_FALSE;
  const cpl_size mindeg[] = { 0, 0 },
                 maxdeg[] = { aXOrder, aYOrder };
  cpl_error_code rc = cpl_polynomial_fit(poly, pos, &sym, val, NULL, CPL_TRUE,
                                         mindeg, maxdeg);
  cpl_matrix_delete(pos);
  cpl_vector_delete(val);

  /* create the output image from the polynomial, if the fit succeeded */
  cpl_image *fit = NULL;
  if (rc == CPL_ERROR_NONE) {
    fit = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
    cpl_image_fill_polynomial(fit, poly, 1, 1, 1, 1);
    if (cpl_image_get_bpm_const(aImage)) {
      cpl_image_reject_from_mask(fit, cpl_image_get_bpm_const(aImage));
    } /* if input has bpm */
  } /* if fit without error */
  cpl_polynomial_delete(poly);
  return fit;
} /* muse_utils_image_fit_polynomial() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Compute centroid over an image window, optionally marginalizing over
            the background.
  @param    aImage    input image to use
  @param    aX1       lower left x position of window
  @param    aY1       lower left y position of window
  @param    aX2       upper right x position of window
  @param    aY2       upper right y position of window
  @param    aX        computed centroid in x-direction
  @param    aY        computed centroid in x-direction
  @param    aBgType   specifies how to handle the background
  @return   CPL_ERROR_NONE on success, another CPL error code on failure

  This computes the ("marginal") centroid over an image window, similar to
  IRAF's imcentroid algorithm but without iterations. Before the normal centroid
  is actually computed, the average level of the image is subtracted. Only the
  positions with positive values are then used in the computation of the output
  centroid.

  @error{return CPL_ERROR_NULL_INPUT, aImage or both aXCen and aYCen are NULL}
  @error{propagate error code,
         cpl_image_extract() fails for the given coordinates}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         aBgType does not contain a valid centroid type}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_utils_image_get_centroid_window(cpl_image *aImage, int aX1, int aY1,
                                     int aX2, int aY2, double *aX, double *aY,
                                     muse_utils_centroid_type aBgType)
{
  cpl_ensure_code(aImage, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aX || aY, CPL_ERROR_NULL_INPUT);

  cpl_image *im = cpl_image_extract(aImage, aX1, aY1, aX2, aY2);
  cpl_ensure_code(im, cpl_error_get_code());

  double bg = 0.; /* subtract the background */
  if (aBgType == MUSE_UTILS_CENTROID_MEAN) {
    bg = cpl_image_get_mean(im);
  } else if (aBgType == MUSE_UTILS_CENTROID_MEDIAN) {
    bg = cpl_image_get_median(im);
  } else {
    cpl_ensure_code(aBgType == MUSE_UTILS_CENTROID_NORMAL,
                    CPL_ERROR_ILLEGAL_INPUT);
  }
  cpl_image_subtract_scalar(im, bg);

  /* centroid in x direction */
  if (aX) {
    cpl_image *row = cpl_image_collapse_create(im, 0);
    double w = 0., /* weight */
           f = 0.; /* flux */
    int i, n = cpl_image_get_size_x(row);
    for (i = 1; i <= n; i++) {
      int err;
      double value = cpl_image_get(row, i, 1, &err);
      if (err || (value < 0 && aBgType != MUSE_UTILS_CENTROID_NORMAL)) {
        continue;
      }
      w += i * value;
      f += value;
    } /* for i */
    *aX = w / f + aX1 - 1;
    cpl_image_delete(row);
  } /* if aX */
  /* centroid in y direction */
  if (aY) {
    cpl_image *col = cpl_image_collapse_create(im, 1);
    double w = 0., /* weight */
           f = 0.; /* flux */
    int j, n = cpl_image_get_size_y(col);
    for (j = 1; j <= n; j++) {
      int err;
      double value = cpl_image_get(col, 1, j, &err);
      if (err || (value < 0 && aBgType != MUSE_UTILS_CENTROID_NORMAL)) {
        continue;
      }
      w += j * value;
      f += value;
    } /* for j */
    *aY = w / f + aY1 - 1;
    cpl_image_delete(col);
  } /* if aY */
  cpl_image_delete(im);

  return CPL_ERROR_NONE;
} /* muse_utils_image_get_centroid_window() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Compute centroid of a two-dimensional dataset.
  @param   aPositions   matrix containing the positions
  @param   aValues      vector containing the data values
  @param   aErrors      vector containing the data errors (1 sigma, can be NULL)
  @param   aX           computed horizontal centroid position
  @param   aY           computed vertical centroid position
  @param   aBgType      specifies how to handle the background
  @return  CPL_ERROR_NONE on success, another cpl_error_code on failure.

  This computes the center of gravity, using the data values and optionally the
  data errors as weights.

  The input aPositions matrix must be for two-dimensional data, i.e. contain
  two columns, and the number of rows must be equal to the number of entries in
  both vectors.

  @error{return CPL_ERROR_NULL_INPUT, aPositions or aValues are NULL}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aPositions does not contain two columns}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aPositions does not give the same number of positions as aValues}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aValues does not contain the same number of positions as aErrors}
  @error{return CPL_ERROR_NULL_INPUT,
         both aX and aY are NULL so that the centroid cannot be returned}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         aBgType does not contain a valid centroid type}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_utils_get_centroid(const cpl_matrix *aPositions,
                        const cpl_vector *aValues, const cpl_vector *aErrors,
                        double *aX, double *aY, muse_utils_centroid_type aBgType)
{
  cpl_ensure_code(aPositions && aValues, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(cpl_matrix_get_ncol(aPositions) == 2, CPL_ERROR_ILLEGAL_INPUT);
  int npoints = cpl_matrix_get_nrow(aPositions);
  cpl_ensure_code(npoints == cpl_vector_get_size(aValues), CPL_ERROR_ILLEGAL_INPUT);
  if (aErrors) {
    cpl_ensure_code(cpl_vector_get_size(aValues) == cpl_vector_get_size(aErrors),
                    CPL_ERROR_ILLEGAL_INPUT);
  }
  cpl_ensure_code(aX || aY, CPL_ERROR_NULL_INPUT);

  const double *values = cpl_vector_get_data_const(aValues);
  double xcen = 0., ycen = 0.,
         weight = 0., bg = 0.;
  if (aBgType == MUSE_UTILS_CENTROID_MEAN) {
    bg = cpl_vector_get_mean(aValues);
  } else if (aBgType == MUSE_UTILS_CENTROID_MEDIAN) {
    bg = cpl_vector_get_median_const(aValues);
  } else {
    cpl_ensure_code(aBgType == MUSE_UTILS_CENTROID_NORMAL,
                    CPL_ERROR_ILLEGAL_INPUT);
  }

  int i;
  for (i = 0; i < npoints; i++) {
    double w = values[i] - bg;
    if (w < 0 && aBgType != MUSE_UTILS_CENTROID_NORMAL) {
      continue;
    }
    if (aErrors) {
      w /= cpl_vector_get(aErrors, i);
    }
    xcen += cpl_matrix_get(aPositions, i, 0) * w;
    ycen += cpl_matrix_get(aPositions, i, 1) * w;
    weight += w;
  } /* for i */
  xcen /= weight;
  ycen /= weight;

  if (aX) {
    *aX = xcen;
  }
  if (aY) {
    *aY = ycen;
  }
  return CPL_ERROR_NONE;
} /* muse_utils_get_centroid() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief   Evaluate a multi-Gaussian with a polynomial background.
  @param   x   The evaluation point
  @param   p   The parameters defining the multi-Gaussian plus polynomial
  @param   f   The function value
  @return  0 on success (failures never occur).

  Parameters are
    ncoeffs
    npeaks
    p(0)
    ...
    p(ncoeffs)
    sigma (the sigma is shared between all Gaussian peaks)
    x(0)
    ...
    x(npeaks - 1)
    A(0)
    ...
    A(npeaks - 1)
 */
/*----------------------------------------------------------------------------*/
static int
muse_utils_multigauss(const double x[], const double p[], double *f)
{
  const double xp = x[0]; /* evaluation point */
  const cpl_size ncoeffs = p[0],
                 npeaks = p[1];
  const double sigma = p[2 + ncoeffs];
  if (sigma == 0.0) { /* special case, Dirac deltas */
    cpl_size i;
    for (i = 0; i < npeaks; i++) {
      if (p[2 + ncoeffs + 1 + i] == xp) {
        *f = DBL_MAX;
        return 0;
      }
    }
    *f = 0.;
    return 0;
  }

  /* compute the function value for the normal case */
  *f = 0.;
  /* evaluate the polynomial, adding all orders to the function value */
  cpl_size ic;
  for (ic = 0; ic < ncoeffs; ic++) {
    *f += p[2 + ic] * pow(xp, ic);
  }
  /* evaluate the Gaussians, adding all peaks to the function value */
  cpl_size ip;
  for (ip = 0; ip < npeaks; ip++) {
    const double xi = p[3 + ncoeffs + ip],
                 Ai = p[3 + ncoeffs + npeaks + ip],
                 exponent = (xi - xp) / sigma;
    *f += Ai / CPL_MATH_SQRT2PI / sigma * exp(-0.5 * exponent*exponent);
  }
#if 0
  printf("eval at %f --> %f\n", xp, *f);
  int i;
  for (i = 0; i < ncoeffs + 2*npeaks + 3; i++) {
    printf("  [%02d] %f\n", i, p[i]);
  }
  fflush(stdout);
#endif
  return 0;
} /* muse_utils_multigauss() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief   Evaluate the partial derivatives of a multi-Gaussian with a
           polynomial background.
  @param   x   The evaluation point
  @param   p   The parameters defining the multi-Gaussian plus polynomial
  @param   f   The values of the partial derivatives
  @return  0 on success (failures never occur).
 */
/*----------------------------------------------------------------------------*/
static int
muse_utils_dmultigauss(const double x[], const double p[], double f[])
{
  const cpl_size ncoeffs = p[0],
                 npeaks = p[1];
  const double sigma = p[2 + ncoeffs];
  if (sigma == 0.0) { /* special case, Dirac deltas */
    memset(f, 0, sizeof(double) * (ncoeffs + 2*npeaks + 3));
    return 0;
  }

  const double xp = x[0]; /* evaluation point */
  f[0] = f[1] = 0.; /* derivatives of the number of parameters are always zero! */
  /* the derivatives by the coefficients of the polynomial */
  cpl_size ic;
  for (ic = 0; ic < ncoeffs; ic++) {
    f[2 + ic] = pow(xp, ic);
  }
  /* derivative regarding sigma */
  f[2 + ncoeffs] = 0.;
  cpl_size ip;
  for (ip = 0; ip < npeaks; ip++) {
    const double xi = p[3 + ncoeffs + ip],
                 Ai = p[3 + ncoeffs + npeaks + ip],
                 exponent = (xi - xp) / sigma,
                 expsq = exponent * exponent,
                 expfunc = exp(-0.5 * expsq);
    f[2 + ncoeffs] -= Ai / CPL_MATH_SQRT2PI / (sigma*sigma)
                    * (1 - expsq) * expfunc;
    /* derivative regarding centers */
    f[3 + ncoeffs + ip] = -Ai / CPL_MATH_SQRT2PI / (sigma*sigma*sigma)
                        * (xi - xp) * expfunc;
    /* derivative regarding fluxes */
    f[3 + ncoeffs + npeaks + ip] = 1 / CPL_MATH_SQRT2PI / sigma * expfunc;
  } /* for ip (peak indices) */
#if 0
  printf("deval at %f -->\n", xp);
  int i;
  for (i = 0; i < ncoeffs + 2*npeaks + 3; i++) {
    printf("  [%02d] %e %f\n", i, f[i], p[i]);
  }
  fflush(stdout);
#endif
  return 0;
} /* muse_utils_dmultigauss() */

/*----------------------------------------------------------------------------*/
/**
  @brief   Carry out a multi-Gaussian fit of data in a vector.
  @param   aX            Positions to fit
  @param   aY            Values and associated errors to fit
  @param   aCenter       centers of the Gaussian peaks
  @param   aSigma        the common Gaussian sigma of all peaks; use a negative
                         value to have sigma be a fixed parameter in the fit
  @param   aFlux         fluxes of the Gaussian peaks
  @param   aPoly         coefficients of the background polynomial
  @param   aMSE          computed mean squared error of the fit
  @param   aRedChisq     reduced chi square of the fit
  @param   aCovariance   output covariance matrix of the fit
  @return  CPL_ERROR_NONE on success, another CPL error code on failure.

  This function fits multiple Gaussians plus a background polynomial to the
  dataset given by aX and aY. The number of Gaussian peaks is determined by the
  length of the vector aCenter. It is mandatory to use it to give first-guess
  values for the centers of the peaks and an estimate of the expected Gaussian
  sigma in aSigma. aFlux has to be of the same length as aCenter, if given its
  values are also taken as first guesses of the areas of the Gaussian peaks
  (the fluxes). The background polynomial is defined using the aPoly vector,
  its contents are first-guess values for the polynomial coefficients, its
  length is used to define the order of the polynomial. If aPoly is NULL, the
  background is assumed to be a constant zero level.

  If successful, fit results are returned in aCenter, aSigma, and -- if given
  -- aFlux. If aSigma was negative on input, the absolute value returned is
  same as the input, but positive. Then, the covariance matrix aCovariance is
  created and filled; it has to be deallocated using cpl_matrix_delete() after
  use. The order of parameters in the covariance matrix is the following:
  polynomial coefficients, sigma, centers, fluxes (where the number of
  elements, except for sigma, is determined by the lengths of the input
  objects).

  This function is loosely modeled after cpl_vector_fit_gaussian(), it also
  uses cpl_fit_lvmq() to do that actual fit. But since computing first-guess
  parameters from the data is not possible in a meaningful way for
  multi-Gaussians, at least not without knowing the number of peaks, this
  function requires first guesses to be given, with the exception of the
  fluxes.

  @error{return CPL_ERROR_NULL_INPUT, aX\, aY\, aCenter\, or aSigma are NULL}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aY contains a different number of points as aX}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aFlux is given but contains a different number of parameters (peaks) than aCenter}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         aRedChisq is not NULL but there are less points than parameters\, so that it cannot be computed}
  @error{propagate error, cpl_fit_lvmq() fails}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_utils_fit_multigauss_1d(const cpl_vector *aX, const cpl_bivector *aY,
                             cpl_vector *aCenter, double *aSigma,
                             cpl_vector *aFlux, cpl_vector *aPoly,
                             double *aMSE, double *aRedChisq,
                             cpl_matrix **aCovariance)
{
  if (aCovariance) {
    *aCovariance = NULL;
  }
  cpl_ensure_code(aX && aY && aCenter && aSigma, CPL_ERROR_NULL_INPUT);
  cpl_size npoints = cpl_vector_get_size(aX);
  cpl_ensure_code(npoints == cpl_bivector_get_size(aY), CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_size npeaks = cpl_vector_get_size(aCenter);
  cpl_ensure_code(!aFlux || npeaks == cpl_vector_get_size(aFlux),
                  CPL_ERROR_INCOMPATIBLE_INPUT);
  cpl_size ncoeffs = aPoly ? cpl_vector_get_size(aPoly) : 0,
           npars = ncoeffs /* poly coeffs */ + 1 /* sigma */
                 + 2 * npeaks /* centers and fluxes */;
  cpl_ensure_code(!aRedChisq || npoints >= npars, CPL_ERROR_ILLEGAL_INPUT);

  /* "transform" the input data into the right structures for cpl_fit_lvmq() */
  cpl_matrix *x = cpl_matrix_wrap(npoints, 1, (double *)cpl_vector_get_data_const(aX));
  const cpl_vector *y = cpl_bivector_get_x_const(aY),
                   *ye = cpl_bivector_get_y_const(aY);
  /* set up and fill the parameters structure */
  cpl_vector *p = cpl_vector_new(npars + 2);
  int *pflags = cpl_calloc(npars + 2, sizeof(int));
  /* first the numbers (of polynomial coefficients and peaks) */
  cpl_vector_set(p, 0, ncoeffs);
  cpl_vector_set(p, 1, npeaks);
  cpl_size i; /* all but these two first parameters participate in the fit */
  for (i = 2; i < npars + 2; i++) {
    pflags[i] = 1; /* fit this parameter, set to non-zero value */
  } /* for i (all but the first two parameters) */
#if 0
  cpl_array *aflags = cpl_array_wrap_int(pflags, npars + 2);
  printf("aflags, non-zero means parameter is fitted by cpl_fit_lvmq():\n");
  cpl_array_dump(aflags, 0, 1000, stdout);
  fflush(stdout);
  cpl_array_unwrap(aflags);
#endif
  /* then the polynomial */
  cpl_size j;
  for (j = 0, i = 2; j < ncoeffs; j++, i++) {
    cpl_vector_set(p, i, cpl_vector_get(aPoly, j));
  }
  /* the common sigma value */
  double sigma = fabs(*aSigma);
  if (*aSigma < 0) {
    pflags[i] = 0; /* fix the sigma parameter */
  }
  cpl_vector_set(p, i++, sigma);
  /* the first-guess centers as passed into this function */
  for (j = 0; j < npeaks; j++, i++) {
    cpl_vector_set(p, i, cpl_vector_get(aCenter, j));
  }
  /* the first-guess fluxes, if passed into this function */
  for (j = 0; j < npeaks; j++, i++) {
    if (aFlux) {
      cpl_vector_set(p, i, cpl_vector_get(aFlux, j));
    } else {
      cpl_vector_set(p, i, 1.);
    }
  }
#if 0
  printf("input parameters p and their pflags:\n");
  for (j = 0; j < cpl_vector_get_size(p); j++) {
    printf("  [%02d] %f   %s\n", j, cpl_vector_get(p, j),
           pflags[j] ? "\tfitted" : "constant");
  }
#endif
  cpl_matrix *covariance = NULL;
  cpl_error_code rc = cpl_fit_lvmq(x, NULL, y, ye, p, pflags,
                                   muse_utils_multigauss, muse_utils_dmultigauss,
                                   CPL_FIT_LVMQ_TOLERANCE, CPL_FIT_LVMQ_COUNT,
                                   CPL_FIT_LVMQ_MAXITER, aMSE, aRedChisq,
                                   &covariance);
  cpl_matrix_unwrap(x);
  cpl_free(pflags);
#if 0
  printf("output parameters vector p (%e, %e):\n",
         aMSE ? sqrt(*aMSE) : 0.0, aRedChisq ? *aRedChisq : 0.0);
  cpl_vector_dump(p, stdout);
  fflush(stdout);
#endif
  /* get all parameters back into the input structures, same order as above */
  for (j = 0, i = 2; j < ncoeffs; j++, i++) {
    cpl_vector_set(aPoly, j, cpl_vector_get(p, i));
  }
  /* In principle, the LM algorithm might have converged to a negative sigma *
   * (even if the guess value was positive). Make sure that the returned     *
   * sigma is positive (by convention), see cpl_vector_fit_gaussian().       */
  *aSigma = fabs(cpl_vector_get(p, i++));
  for (j = 0; j < npeaks; j++, i++) {
    cpl_vector_set(aCenter, j, cpl_vector_get(p, i));
  }
  /* the first-guess fluxes, if passed into this function */
  if (aFlux) {
    for (j = 0; j < npeaks; j++, i++) {
      cpl_vector_set(aFlux, j, cpl_vector_get(p, i));
    }
  }
  /* extract the relevant part of the covariance matrix, if needed       *
   * (the number of coefficients and peaks are not relevant parameters!) */
  if (aCovariance) {
    *aCovariance = cpl_matrix_extract(covariance, 2, 2, 1, 1,
                                      cpl_matrix_get_nrow(covariance) - 2,
                                      cpl_matrix_get_ncol(covariance) - 2);
  }
  cpl_matrix_delete(covariance);
  cpl_vector_delete(p);
  return rc;
} /* muse_utils_fit_multigauss_1d() */

#if MOFFAT_USE_MUSE_OPTIMIZE
/* structure to pass around data needed for the *
 * evaluation using muse_moffat_2d_optfunc()    */
typedef struct {
  const cpl_matrix *positions;
  const cpl_vector *values;
  const cpl_vector *errors;
} fitdata_t;

/*---------------------------------------------------------------------------*/
/**
  @private
  @brief   Compute residuals of an elliptical 2D Moffat function.
  @param   aData        data needed for the evaluation (of type fitdata *)
  @param   aParams      parameters defining the polynomial (array of 8 values)
  @param   aResiduals   the output residuals array
  @return  0 on success

  The aParams elements are in the same order as the aParams[] elements of
  muse_utils_fit_moffat_2d().
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_moffat_2d_optfunc(void *aData, cpl_array *aParams, cpl_array *aResiduals)
{
  const cpl_matrix *pos = ((fitdata_t *)aData)->positions;
  const cpl_vector *val = ((fitdata_t *)aData)->values,
                   *err = ((fitdata_t *)aData)->errors;
  /* Compute function residuals */
  const double *p = cpl_array_get_data_double_const(aParams);
  double *residuals = cpl_array_get_data_double(aResiduals);
  int i, npoints = cpl_vector_get_size(val);
  for (i = 0; i < npoints; i++) {
    double xterm = (cpl_matrix_get(pos, i, 0) - p[2]) / p[4],
           yterm = (cpl_matrix_get(pos, i, 1) - p[3]) / p[5],
           crossterm = 2 * p[7] * xterm * yterm,
           base = 1 + (xterm*xterm + crossterm + yterm*yterm) / (1 + p[7]*p[7]),
           moffat = p[0] + p[1] * (p[6] - 1)
                  / (CPL_MATH_PI * p[4]*p[5] * sqrt(1 - p[7]*p[7]))
                  * pow(base, -p[6]);
    /* set to the actual value */
    residuals[i] = cpl_vector_get(val, i) - moffat;
    /* finally weight by the error on this point */
    residuals[i] /= cpl_vector_get(err, i);
  } /* for i (all points) */
  return CPL_ERROR_NONE;
} /* muse_moffat_2d_optfunc() */

#else /* MOFFAT_USE_MUSE_OPTIMIZE follows */

/*---------------------------------------------------------------------------*/
/**
  @private
  @brief   Evaluate an elliptical 2D Moffat function.
  @param   xy   The evaluation point (array of two values)
  @param   p    The parameters defining the polynomial (array of 8 values)
  @param   f    The function value
  @return  0 on success

  The p[] elements are in the same order as the aParams[] elements of
  muse_utils_fit_moffat_2d().
 */
/*---------------------------------------------------------------------------*/
static int
muse_moffat_2d_function(const double xy[], const double p[], double *f)
{
  double xterm = (xy[0] - p[2]) / p[4], /* xy[0] is x */
         yterm = (xy[1] - p[3]) / p[5], /* xy[1] is y */
         crossterm = 2 * p[7] * xterm * yterm,
         rhoterm = 1. - p[7]*p[7],
         base = 1. + (xterm*xterm + crossterm + yterm*yterm) / rhoterm;
  *f = p[0] + p[1] * (p[6] - 1.) / (CPL_MATH_PI * p[4]*p[5] * sqrt(rhoterm))
     * pow(base, -p[6]);
//printf("%s(%f,%f [%e %e %e %e %e %e %e %e]) = %e\n", __func__, xy[0], xy[1], p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], *f);
//fflush(stdout);
  return 0;
} /* muse_moffat_2d_function() */

/*---------------------------------------------------------------------------*/
/**
  @private
  @brief   Evaluate the derivatives of an elliptical 2D Moffat function.
  @param   xy   The evaluation point (array of two values)
  @param   p    The parameters defining the polynomial (array of 8 values)
  @param   f    The values of the derivative (an array of 8 values)
  @return  0 on success

  The p[] elements are in the same order as the aParams[] elements of
  muse_utils_fit_moffat_2d().
 */
/*---------------------------------------------------------------------------*/
static int
muse_moffat_2d_derivative(const double xy[], const double p[], double f[])
{
  double xterm = (xy[0] - p[2]) / p[4], /* xy[0] is x */
         yterm = (xy[1] - p[3]) / p[5], /* xy[1] is y */
         crossterm = 2 * p[7] * xterm * yterm,
         rhoterm = 1. - p[7]*p[7],
         base = 1. + (xterm*xterm + crossterm + yterm*yterm) / (rhoterm);
  f[0] = 1;                                                     /* dM(x,y)/dB */
  f[1] = (p[6] - 1.) / (CPL_MATH_PI * p[4]*p[5] * sqrt(rhoterm))/* dM(x,y)/dA */
       * pow(base, -p[6]);
  f[2] = 2 * p[1] * p[6]*(p[6] - 1.)                           /* dM(x,y)/dxc */
       / (CPL_MATH_PI * p[4]*p[4] * p[5] * pow(rhoterm, 3./2.))
       * (xterm + p[7] * yterm) * pow(base, -p[6]-1.);
  f[3] = 2 * p[1] * p[6]*(p[6] - 1.)                           /* dM(x,y)/dyc */
       / (CPL_MATH_PI * p[4] * p[5]*p[5] * pow(rhoterm, 3./2.))
       * (yterm + p[7] * xterm) * pow(base, -p[6]-1.);
  f[4] = p[1] * (p[6] - 1.)                                /* dM(x,y)/dalphax */
       / (CPL_MATH_PI * p[4]*p[4] * p[5] * sqrt(rhoterm))
       * (-pow(base, -p[6]) + 2 * p[6] / (rhoterm) * pow(base, -p[6]-1.)
                              * (xterm*xterm + 0.5*crossterm));
  f[5] = p[1] * (p[6] - 1.)                                /* dM(x,y)/dalphay */
       / (CPL_MATH_PI * p[4] * p[5]*p[5] * sqrt(rhoterm))
       * (-pow(base, -p[6]) + 2 * p[6] / (rhoterm) * pow(base, -p[6]-1.)
                              * (yterm*yterm + 0.5*crossterm));
  f[6] = p[1] / (CPL_MATH_PI * p[4]*p[5] * sqrt(rhoterm))    /* dM(x,y)/dbeta */
       * pow(base, -p[6]) * (1. + (p[6] - 1.) * log(base));
  f[7] = p[1] * (p[6] - 1.)                                   /* dM(x,y)/drho */
       / (CPL_MATH_PI * p[4]*p[5] * pow(rhoterm, 3./2.))
       * (p[7] * pow(base, -p[6])
          - 2. * p[6] * pow(base, -p[6]-1.)
            * (xterm*yterm * (1 + 2*p[7]*p[7] / rhoterm)
               + p[7] / rhoterm * (xterm*xterm + yterm*yterm) ));
//printf("%s = %e, %e, %e, %e, %e, %e, %e, %e\n", __func__, f[0], f[1], f[2], f[3], f[4], f[5], f[6], f[7]);
//fflush(stdout);
  return 0;
} /* muse_moffat_2d_derivative() */

#endif

/*----------------------------------------------------------------------------*/
/**
  @brief    Fit a 2D Moffat function to a given set of data.
  @param    aPositions   input image to use for the fit
  @param    aValues      input values to use for the fit
  @param    aErrors      input errors to use for the fit (can be NULL)
  @param    aParams      parameter array (type double)
  @param    aPErrors     parameter error double array (can be NULL)
  @param    aPFlags      parameter flags integer array (can be NULL)
  @param    aRMS         root mean square of the fit (can be NULL)
  @param    aRedChisq    reduced chi^2 of the fit (can be NULL)
  @return   CPL_ERROR_NONE on success any other CPL error code on failure.

  This fits a Moffat (1969 A&A 3, 455, Eq. 7) function in two-dimensional
  cartesian form allowing for non-circular isophotes
  \f[
    \mathrm{moffat}(x,y) = B + A \frac{\beta - 1} {\pi \alpha_x \alpha_y \sqrt{1 - \rho^2}}
                  \left[ 1 + \frac{ \left(\frac{x - x_c} {\alpha_x}\right)^2
                         + 2 \rho \frac{(x-x_c)(y-y_c)} {\alpha_x \alpha_y}
                         + \left(\frac{y - y_c} {\alpha_y}\right)^2} {1 - \rho^2}
                  \right]^{-\beta}
  \f]
  to the input data. The parameters and their starting values are

  - aParams[0]: \f$B\f$, constant background level; median of the dataset
  - aParams[1]: \f$A\f$, volume of the function (flux); integrated value of the
                dataset in excess of the median
  - aParams[2]: \f$x_c\f$, horizontal center; marginal centroid of the dataset
  - aParams[3]: \f$y_c\f$, vertical center; see \f$x_c\f$
  - aParams[4]: \f$\alpha_x\f$, related to horizontal HWHM; computed from the HWHM of
                the dataset, using at least three data values near the half peak
                around the (marginal) centroid
  - aParams[5]: \f$\alpha_y\f$, related to vertical HWHM; see \f$\alpha_x\f$
  - aParams[6]: \f$\beta\f$, seeing-related Moffat parameter; 2.5
  - aParams[7]: \f$\rho\f$, the x-y correlation parameter; 0.0

  Valid inputs in aParams override the first-guess parameters given above. If
  only one valid alpha parameter was given, the other one is used as starting
  value.

  In case of uniform data, no error will be returned but a flat Moffat function
  with undefined center and widths will be created.

  References:
  - Wikipedia
    http://en.wikipedia.org/w/index.php?title=Moffat_distribution&oldid=347629009
  - This function uses quite a bit of the error checking and initial setup code
    as well as the ellipticity implementation of cpl_fit_image_gaussian(),
    written by Carlo Izzo, (C) 2001-2008 European Southern Observatory.
  - MPFIT2DPEAK IDL routine by Craig B. Markwardt (NASA/GSFC)
    http://spectro.princeton.edu/idlutils_doc.html#MPFIT2DPEAK
    None of his code was actually used for this implementation, but the
    definition of the different functions helped to define the parameters.

  @error{return CPL_ERROR_NULL_INPUT, aPositions\, aValues\, or aParams are NULL}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aPositions does not give the same number of positions as aValues}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aPositions does not contain two columns}
  @error{return CPL_ERROR_INCOMPATIBLE_INPUT,
         aErrors is given but does not have the same size as aValues}
  @error{return CPL_ERROR_INVALID_TYPE, aParams is not of type double}
  @error{return CPL_ERROR_INVALID_TYPE,
         aPErrors is given but not of type double}
  @error{return CPL_ERROR_INVALID_TYPE, aPFlags is given but not of type int}
  @error{return CPL_ERROR_DATA_NOT_FOUND,
         aPErrors and/or aRedChisq are given but aErrors is NULL}
  @error{return CPL_ERROR_SINGULAR_MATRIX, input data has less than 8 positions}
  @error{return CPL_ERROR_ILLEGAL_INPUT, aPFlags did not leave any parameter free}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         aPFlags was set to freeze a parameter but this parameter does not have an input value in aParams}
  @error{propagate return code of cpl_fit_lvmq(), the actual fit fails}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_utils_fit_moffat_2d(const cpl_matrix *aPositions,
                         const cpl_vector *aValues, const cpl_vector *aErrors,
                         cpl_array *aParams, cpl_array *aPErrors,
                         const cpl_array *aPFlags,
                         double *aRMS, double *aRedChisq)
{
  /* lots of error checking, almost exactly taken from CPL */
  if (!aPositions) {
    return cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT,
                                 "Missing input positions.");
  }
  if (!aValues) {
    return cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT,
                                 "Missing input values / errors.");
  }
  if (cpl_matrix_get_ncol(aPositions) != 2) {
    return cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                                 "Input positions are not for two-dimensional data.");
  }
  if (cpl_vector_get_size(aValues) != cpl_matrix_get_nrow(aPositions)) {
    return cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                                 "Input positions and values data must have same size.");
  }
  if (aErrors && (cpl_vector_get_size(aValues) != cpl_vector_get_size(aErrors))) {
    return cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                                 "Input vectors must have same size.");
  }
  if (!aParams) {
    return cpl_error_set_message(__func__, CPL_ERROR_NULL_INPUT,
                                 "Missing input parameters array.");
  }
  if (cpl_array_get_type(aParams) != CPL_TYPE_DOUBLE) {
    return cpl_error_set_message(__func__, CPL_ERROR_INVALID_TYPE,
                                 "Parameters array should be CPL_TYPE_DOUBLE.");
  }
  if (aPErrors && (cpl_array_get_type(aPErrors) != CPL_TYPE_DOUBLE)) {
    return cpl_error_set_message(__func__, CPL_ERROR_INVALID_TYPE,
                                 "Parameters error array should be CPL_TYPE_DOUBLE.");
  }
  if (aPFlags && (cpl_array_get_type(aPFlags) != CPL_TYPE_INT)) {
    return cpl_error_set_message(__func__, CPL_ERROR_INVALID_TYPE,
                                 "Parameters error array should be CPL_TYPE_INT.");
  }
  if ((aPErrors || aRedChisq) && !aErrors) {
    return cpl_error_set_message(__func__, CPL_ERROR_DATA_NOT_FOUND,
                                 "Missing input parameters errors.");
  }
  int npoints = cpl_matrix_get_nrow(aPositions);
  if (npoints < 8) {
    /* Too few positions for 8 free parameters!  */
    return cpl_error_set_message(__func__, CPL_ERROR_SINGULAR_MATRIX,
                                 "%d are not enough points to fit a Moffat profile.",
                                 npoints);
  }

  int pflags[8] = { 1, 1, 1, 1, 1, 1, 1, 1 };
  /* Ensure that frozen parameters have a value (first-guess) */
  if (aPFlags) {
      int idx, nparam = 0;
      for (idx = 0; idx < 8; idx++) {
        int err, flag = cpl_array_get_int(aPFlags, idx, &err);
        if (err || flag) {
          continue;
        }
        pflags[idx] = 0; /* Flag it as frozen */
        nparam++;
        cpl_array_get_double(aParams, idx, &err);
        if (err) {
          return cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT,
                                       "Missing frozen value for parameter %d.", idx);
        }
      } /* for idx (all parameters) */
      /* Ensure that not all parameters are frozen */
      if (nparam == 8) {
        return cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT,
                                     "No free parameters");
      }
  } /* if aPFlags */

  /* Determine first-guess for gaussian parameters. Check if        *
   * provided by caller - if not build own guesses...               *
   *                                                                *
   * 0) Background level: if not given taken as median value within *
   *    fitting domain. It can be negative...                       */
  int invalid;
  double bg = cpl_array_get_double(aParams, 0, &invalid);
  if (invalid) {
    bg = cpl_vector_get_median_const(aValues);
  }

  /* 1) Volume is really set later-on. Here is just a quick estimate, to know *
   *    whether there is a peak or a hole. If it is flat, leave quickly...    */
  double volguess = (cpl_vector_get_mean(aValues) - bg) * npoints;
  if (fabs(volguess) < FLT_EPSILON) {
    /* Data are flat: return a flat Moffat, with undefined center and widths */
    cpl_array_set_double(aParams, 0, bg);
    cpl_array_set_double(aParams, 1, 0.);
    cpl_array_set_invalid(aParams, 2);
    cpl_array_set_invalid(aParams, 3);
    cpl_array_set_invalid(aParams, 4);
    cpl_array_set_invalid(aParams, 5);
    cpl_array_set_double(aParams, 6, 1.); /* beta = 1 --> Moffat = 0 */
#if 0
    printf("flat Moffat:\n");
    cpl_array_dump(aParams, 0, 10, stdout);
    fflush(stdout);
#endif
    return CPL_ERROR_NONE;
  }

  /* 2), 3) Position of center. Compute it as the centroid if not given. *
   *        This likely does not waste time, as this center is also used *
   *        to determine the alpha parameters below.                     */
  double xcen, ycen;
  muse_utils_get_centroid(aPositions, aValues, aErrors, &xcen, &ycen,
                          MUSE_UTILS_CENTROID_MEDIAN);
  double xc = cpl_array_get_double(aParams, 2, &invalid);
  if (invalid) {
    xc = xcen;
  }
  double yc = cpl_array_get_double(aParams, 3, &invalid);
  if (invalid) {
    yc = ycen;
  }

  /* 6) Beta, seeing related Moffat parameter: if not given by the user, *
   *    it is set to 2.5, which seems to be typical for stellar images.  *
   *    There does not seem to be a simple way to estimate it from the   *
   *    data, independently of the alphax,alphay parameters.             */
  double beta = cpl_array_get_double(aParams, 6, &invalid);
  if (invalid) {
    beta = 2.5;
  }

  /* 4), 5) Widths: if neither alphax nor alphay are given by the caller the  *
   *        estimate for both is the radius of an approximate half-peak point *
   *        from the peak point. Very rough, but accuracy is not an issue at  *
   *        this stage, just need a rough starting value. If only one width   *
   *        is given by the caller, the other one is set to the same value.   */
  double alphay, alphax = cpl_array_get_double(aParams, 4, &invalid);
  if (invalid) {
    alphay = cpl_array_get_double(aParams, 5, &invalid);
    if (invalid) {
      double amplitude = 0.;
      if (volguess > 0.) {
        amplitude = cpl_vector_get_max(aValues) - bg;
      } else {
        amplitude = cpl_vector_get_min(aValues) - bg;
      }
      double halfpeak = amplitude / 2. + bg,
             limit = amplitude * 0.01; /* 1% accuracy for the start */
      const double *values = cpl_vector_get_data_const(aValues);
      cpl_vector *vradius = cpl_vector_new(1); /* store radius values */
      int i, nfound;
      do {
        nfound = 0;
        for (i = 0; i < npoints; i++) {
          if (values[i] > halfpeak - limit && values[i] < halfpeak + limit) {
            cpl_vector_set_size(vradius, nfound + 1);
            double radius = sqrt(pow(cpl_matrix_get(aPositions, i, 0) - xcen, 2)
                                 + pow(cpl_matrix_get(aPositions, i, 1) - ycen, 2));
#if 0
            printf("radius(%d, %d) = %f\n", i, nfound+1, radius);
#endif
            cpl_vector_set(vradius, nfound++, radius);
          }
        } /* for i (all values) */
        /* if we go through this loop again, we will find more points and *
         * so all previous vector entries will be completely overwritten  */
        limit *= 2; /* be twice as tolerant to find the points next time */
#if 0
        printf("found %d points (limit = %f)\n", nfound, limit / 2);
#endif
      } while (nfound < 3 && isfinite(limit));
      if (!isfinite(limit)) {
        /* if the data is so weird that we don't find points within *
         * finite limit, then any alpha is a good first guess...    */
        alphax = alphay = 1.;
      } else {
        /* the radius of found points from 1st-guess center is the *
         * HWHM, the alpha parameter of a Moffat function is then *
         *   alpha = HWHM / sqrt(2^(1/beta) - 1)                  */
        alphax = alphay = cpl_vector_get_mean(vradius) / sqrt(pow(2., 1./beta)-1);
#if 0
        printf("estimated alphas = %f from radius %f\n", alphax, cpl_vector_get_mean(vradius));
        fflush(stdout);
#endif
      }
      cpl_vector_delete(vradius);
    } else {
      alphax = alphay;
    }
  } else {
    alphay = cpl_array_get_double(aParams, 5, &invalid);
    if (invalid) {
      alphay = alphax;
    }
  }

  /* 1) Volume. If not given by the user, it is derived    *
   *    from the max (min) value of the data distribution. */
  double volume = cpl_array_get_double(aParams, 1, &invalid);
  if (invalid) {
    /* The above seems to be a good enough first-guess.    *
     * Deriving a better guess from the amplitude would    *
     * require more solid guesses of the other parameters. */
    volume = volguess;
  }

  /* 7) Rho, x-y correlation parameter: if not given by the user,  *
   *    it is set to zero (no correlation, i.e. circular Moffat or *
   *    one that is elongated along x or y).                       */
  double rho = cpl_array_get_double(aParams, 7, &invalid);
  if (invalid) {
    rho = 0.;
  }

  /* Yay! Now all parameters are set to initial values, and we can do the fit! */
  cpl_vector *params = cpl_vector_new(8);
  cpl_vector_set(params, 0, bg);
  cpl_vector_set(params, 1, volume);
  cpl_vector_set(params, 2, xc);
  cpl_vector_set(params, 3, yc);
  cpl_vector_set(params, 4, alphax);
  cpl_vector_set(params, 5, alphay);
  cpl_vector_set(params, 6, beta);
  cpl_vector_set(params, 7, rho);
#if 0
  printf("initial guess values for Moffat (vol %e):\n", (cpl_vector_get_mean(aValues) - bg) * npoints);
  cpl_vector_dump(params, stdout);
  fflush(stdout);
#endif
  cpl_matrix *covariance = NULL;

  cpl_error_code rc = CPL_ERROR_NONE;
#if MOFFAT_USE_MUSE_OPTIMIZE
  fitdata_t fitdata;
  fitdata.positions = aPositions;
  fitdata.values = aValues;
  fitdata.errors = aErrors;
  cpl_array *optparams = cpl_array_wrap_double(cpl_vector_get_data(params), 8);
  rc = muse_cpl_optimize_lvmq(&fitdata, optparams, npoints,
                              muse_moffat_2d_optfunc, NULL);
  cpl_array_unwrap(optparams);
#else /* MOFFAT_USE_MUSE_OPTIMIZE follows */
  cpl_errorstate prestate = cpl_errorstate_get();
  rc = cpl_fit_lvmq(aPositions, NULL, aValues, aErrors, params, pflags,
                    muse_moffat_2d_function, muse_moffat_2d_derivative,
                    CPL_FIT_LVMQ_TOLERANCE, CPL_FIT_LVMQ_COUNT,
                    CPL_FIT_LVMQ_MAXITER, aRMS,
                    aErrors ? aRedChisq : NULL, aErrors ? &covariance : NULL);
#endif
  if (aRMS) {
    *aRMS = sqrt(*aRMS);
  }

#if 0
  printf("Moffat fit (rc=%d, %s):\n", rc, cpl_error_get_message());
  cpl_vector_dump(params, stdout);
  fflush(stdout);
#endif
  if (rc == CPL_ERROR_NONE || rc == CPL_ERROR_SINGULAR_MATRIX ||
      rc == CPL_ERROR_CONTINUE) {
    /* The LM algorithm converged. The computation of the covariance  *
     * matrix might have failed. All the above errors must be ignored *
     * because of ticket DFS06126.                                    */

    /* check whether the result makes sense at all... (unlike CPL we use C99) */
    if (isfinite(cpl_vector_get(params, 0)) &&
        isfinite(cpl_vector_get(params, 1)) &&
        isfinite(cpl_vector_get(params, 2)) &&
        isfinite(cpl_vector_get(params, 3)) &&
        isfinite(cpl_vector_get(params, 4)) &&
        isfinite(cpl_vector_get(params, 5)) &&
        isfinite(cpl_vector_get(params, 6)) &&
        isfinite(cpl_vector_get(params, 7))) {
      /* Betting all errors are really to be ignored... */
      rc = CPL_ERROR_NONE;
      cpl_errorstate_set(prestate);

      /* Save best fit parameters: image coordinates, evaluations   *
       * of widths are forced positive (they might be both negative *
       * -- it would generate the same profile).                    */
      cpl_array_set_double(aParams, 0, cpl_vector_get(params, 0));
      cpl_array_set_double(aParams, 1, cpl_vector_get(params, 1));
      cpl_array_set_double(aParams, 2, cpl_vector_get(params, 2));
      cpl_array_set_double(aParams, 3, cpl_vector_get(params, 3));
      cpl_array_set_double(aParams, 4, fabs(cpl_vector_get(params, 4)));
      cpl_array_set_double(aParams, 5, fabs(cpl_vector_get(params, 5)));
      cpl_array_set_double(aParams, 6, cpl_vector_get(params, 6));
      cpl_array_set_double(aParams, 7, cpl_vector_get(params, 7));

      if (aErrors && covariance) {
        int idx;
        for (idx = 0; idx < 8; idx++) {
          if (pflags[idx] && aPErrors) {
             cpl_array_set_double(aPErrors, idx,
                                  sqrt(cpl_matrix_get(covariance, idx, idx)));
          } /* if */
        } /* for idx (all parameters) */
      } /* if */

      /* we don't care about the physical parameters like CPL does, so this was it :-) */
    } /* if isfinite() */
  } /* if rc */
  cpl_matrix_delete(covariance);
  cpl_vector_delete(params);

  return rc;
} /* muse_utils_fit_moffat_2d() */

/*----------------------------------------------------------------------------*/
/**
  @brief  Iterate a polynomial fit.
  @param  aPos     matrix with input positions
  @param  aVal     vector with input values
  @param  aErr     vector with input errors (optional, currently unused)
  @param  aExtra   table with extra information
                   (optional, same number of rows as aVal)
  @param  aOrder   polynomial order to use for the fit
  @param  aRSigma  rejection sigma
  @param  aMSE     output mean squared error of final fit (optional)
  @param  aChiSq   output chi square of final fit (optional)
  @return CPL_ERROR_NONE on success any other CPL error code on failure.

  The position matrix aPos contains the same number of columns as the values
  vector aVal. The number of rows in aPos defines the number of dimensions of
  the output polynomial.

  The table aExtra contains extra information, with each matrix row related to
  each entry in the aVal vector.

  Infinite entries (NANs and INFs) in aVal are rejected at the start, so can be
  used as "bad pixel" markers. The procedure will fail, if only infinite values
  are present.

  @note All input structures (i.e. aPos, aVal, aErr, and aExtra) are changed by
        this routine, so that entries rejected during the iterations are erased.
        This means that pointers to the structures may be changed afterwards.

  If an error occurs, both aChiSq and aMSE are set to high values (DBL_MAX).

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aPos and/or aVal are NULL}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return NULL,
         the number of aPos does not match the size of aVal}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return NULL,
         aErr was specified but does not match the size of aVal}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return NULL,
         aExtra was specified but the number of its rows does not match the size of aVal}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL,
         aVal only contains infinite values}
  @error{set CPL_ERROR_ILLEGAL_OUTPUT\, return NULL,
         the iterative procedure tries to remove the last vector/matrix element}
 */
/*----------------------------------------------------------------------------*/
cpl_polynomial *
muse_utils_iterate_fit_polynomial(cpl_matrix *aPos, cpl_vector *aVal,
                                  cpl_vector *aErr, cpl_table *aExtra,
                                  const unsigned int aOrder, const double aRSigma,
                                  double *aMSE, double *aChiSq)
{
  /* pre-fill diagnistics to high values in case of errors */
  if (aMSE) {
    *aMSE = DBL_MAX;
  }
  if (aChiSq) {
    *aChiSq = DBL_MAX;
  }
  cpl_ensure(aPos && aVal, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(cpl_matrix_get_ncol(aPos) == cpl_vector_get_size(aVal),
             CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  if (aErr) {
    cpl_ensure(cpl_vector_get_size(aVal) == cpl_vector_get_size(aErr),
               CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  }
  if (aExtra) {
    cpl_ensure(cpl_vector_get_size(aVal) == cpl_table_get_nrow(aExtra),
               CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  }

  /* XXX erase positions with NAN values upfront */
  int idx;
  for (idx = 0; idx < cpl_vector_get_size(aVal); idx++) {
    /* compare this residual value to the RMS value */
    if (isfinite(cpl_vector_get(aVal, idx))) {
      continue; /* want to keep all finite numbers */
    }
   /* guard against removing the last element */
    if (cpl_vector_get_size(aVal) == 1) {
      cpl_msg_warning(__func__, "Input vector only contained non-finite elements!");
      break;
    }
    /* remove bad element from the fit structures... */
    cpl_matrix_erase_columns(aPos, idx, 1);
    muse_cplvector_erase_element(aVal, idx);
    if (aErr) { /* check to not generate CPL error */
      muse_cplvector_erase_element(aErr, idx);
    }
    /* ...and from the input matrix, if it's there */
    if (aExtra) {
      cpl_table_erase_window(aExtra, idx, 1);
    }
    idx--; /* we stay at this position to see what moved here */
  } /* for idx */

  /* create the polynomial, using matrix rows to determine dimensions */
  int ndim = cpl_matrix_get_nrow(aPos);
  cpl_polynomial *fit = cpl_polynomial_new(ndim);
  int large_residuals = 1; /* init to force a first fit */
  while (large_residuals > 0) {
    const cpl_boolean sym = CPL_FALSE;
    cpl_size *mindeg = cpl_calloc(ndim, sizeof(cpl_size)),
             *maxdeg = cpl_malloc(ndim * sizeof(cpl_size));
    int i;
    for (i = 0; i < ndim; i++) {
      maxdeg[i] = aOrder;
    }
    cpl_error_code rc = cpl_polynomial_fit(fit, aPos, &sym, aVal, NULL,
                                           CPL_FALSE, mindeg, maxdeg);
    cpl_free(mindeg);
    cpl_free(maxdeg);
    const cpl_size coeff = 0;
    if (rc != CPL_ERROR_NONE || !isfinite(cpl_polynomial_get_coeff(fit, &coeff))) {
      /* don't try to recover from an error, instead clean up and return NULL */
      cpl_errorstate prestate = cpl_errorstate_get();
#if 0
      printf("%s: output polynomial:\n", __func__);
      cpl_polynomial_dump(fit, stdout);
      printf("%s: positions and values that we tried to fit:\n", __func__);
      cpl_matrix_dump(aPos, stdout);
      cpl_vector_dump(aVal, stdout);
      fflush(stdout);
#endif
      cpl_polynomial_delete(fit);
      /* make sure to expose the important error to the caller */
      if (!cpl_errorstate_is_equal(prestate)) {
        cpl_errorstate_set(prestate);
      }
      return NULL;
    }

    /* compute residuals and mean squared error */
    cpl_vector *res = cpl_vector_new(cpl_vector_get_size(aVal));
    cpl_vector_fill_polynomial_fit_residual(res, aVal, NULL, fit, aPos, aChiSq);
    double rms = sqrt(cpl_vector_product(res, res) / cpl_vector_get_size(res));
    if (rms == 0.) { /* otherwise everything will be rejected! */
      rms = DBL_MIN;
    }

#if 0
    printf("%s: polynomial fit (RMS %g chisq %g aRSigma %f -> limit %g):\n",
           __func__, rms, aChiSq ? *aChiSq : 0., aRSigma, aRSigma * rms);
    cpl_polynomial_dump(fit, stdout);
    fflush(stdout);
    char *title = cpl_sprintf("set title \"%s: RMS %g\"\n"
                              "unset key\n", __func__, rms);
    cpl_plot_vector(title, "", "", res);
    cpl_free(title);
#endif

    large_residuals = 0;
    for (i = 0; i < cpl_vector_get_size(res); i++) {
      /* compare this residual value to the RMS value */
      if (fabs(cpl_vector_get(res, i)) < (aRSigma * rms)) {
        /* good fit at this point */
        continue;
      }

      /* bad residual, remove element, from residuals vector and the data vectors */
#if 0
      cpl_msg_debug(__func__, "residual %f RMS %f aRSigma %f -> limit %f",
                    cpl_vector_get(res, i), rms, aRSigma, aRSigma * rms);
#endif
      /* guard against removing the last element */
      if (cpl_vector_get_size(res) == 1) {
        cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "tried to "
                              "remove the last vector/matrix element when "
                              "checking fit residuals (residual %g RMS %g "
                              "aRSigma %f -> limit %g)", cpl_vector_get(res, i),
                              rms, aRSigma, aRSigma * rms);
        cpl_polynomial_delete(fit);
        fit = NULL;
        rms = sqrt(DBL_MAX); /* so that aMSE gets to be DBL_MAX below */
        if (aChiSq) {
          *aChiSq = DBL_MAX;
        }
        large_residuals = 0; /* don't try to fit again */
        break;
      }
      /* remove bad element from the fit structures... */
      muse_cplvector_erase_element(res, i);
      cpl_matrix_erase_columns(aPos, i, 1);
      muse_cplvector_erase_element(aVal, i);
      if (aErr) { /* check to not generate CPL error */
        muse_cplvector_erase_element(aErr, i);
      }
      /* ...and from the input matrix, if it's there */
      if (aExtra) {
        cpl_table_erase_window(aExtra, i, 1);
      }
      large_residuals++;
      i--; /* we stay at this position to see what moved here */
    } /* for i */
    cpl_vector_delete(res);
    if (aMSE) {
      *aMSE = rms * rms;
    }
  } /* while large_residuals */

  return fit;
} /* muse_utils_iterate_fit_polynomial() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Fit a 1D Gaussian to a given wavelength range in a pixel table.
  @param    aPixtable    input pixel table
  @param    aLambda      the wavelength around which to extract the range
  @param    aHalfWidth   the half width of the range in wavelength to use
  @param    aBinSize     the size of the bins in wavelength for the spectrum
  @param    aLo          low sigma-clipping limit for the spectrum
  @param    aHi          high sigma-clipping limit for the spectrum
  @param    aIter        number of iterations for sigma-clipping the spectrum
  @param    aResults     parameter error double array (can be NULL)
  @param    aErrors      parameter flags integer array (can be NULL)
  @return   the center of the Gaussian on success or 0. on error.

  This fits a 1D Gaussian of the form

  @verbatim
    gauss(x) = A / sqrt(2 pi sigma^2) * exp( -(x - xc)^2/(2 sigma^2)) + B@endverbatim

  to the input pixel table. It does this by resampling the pixel table in the
  specified wavelength range into a temporary spectrum, sampled at aBinSize,
  using @ref muse_resampling_spectrum(), which is then given to @c
  cpl_vector_fit_gaussian().

  This function directly returns the center @em xc of the Gaussian fit. If the
  other properties are required, the optional inputs aResults (for the values)
  and aErrors (for the sigma values) need to be passed as CPL arrays of type
  CPL_TYPE_DOUBLE. These arrays are resized to 4 elements and overwritten with
  output. The Gaussian parameters are always written to aResults (if given), the
  errors returned in aErrors may be invalid, except for the 0th element (the
  centroid error) which is always written. The order of the elements is:

  - aParams[0]: xc, the Gaussian center
  - aParams[1]: sigma
  - aParams[2]: A, the area under the Gaussian curve
  - aParams[3]: B, the background level

  First-guess parameters cannot be given, but the wavelength range should be
  given to restrict the data to a set of data that contains one Gaussian-like
  peak.

  @note The input pixel table is preserved, except for the table selection
        flags, which are reset to none selected when the function returns.

  @error{set CPL_ERROR_NULL_INPUT\, return 0.,
         aPixtable or one of its components is NULL}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return 0.,
         aPixtable does not have data in the range |aLambda| +/- aHalfWidth}
  @error{propagate error code\, return 0., muse_resampling_spectrum() fails}
 */
/*----------------------------------------------------------------------------*/
double
muse_utils_pixtable_fit_line_gaussian(muse_pixtable *aPixtable, double aLambda,
                                      double aHalfWidth, double aBinSize,
                                      float aLo, float aHi, unsigned char aIter,
                                      cpl_array *aResults, cpl_array *aErrors)
{
  cpl_ensure(aPixtable && aPixtable->table && aPixtable->header,
             CPL_ERROR_NULL_INPUT, 0.);

  /* make sure we are dealing with positive wavelengthts for comparison *
   * with the pixel table; split the sign off into a separate variable  *
   * that determines, if we flip the spectrum before doing the fit      */
  cpl_boolean flip = aLambda < 0.;
  double lambda = fabs(aLambda);

  /* select (only) the relevant part of the pixel table */
  cpl_table_unselect_all(aPixtable->table);
  cpl_table_or_selected_float(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                              CPL_NOT_LESS_THAN, lambda - aHalfWidth);
  cpl_table_and_selected_float(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                               CPL_NOT_GREATER_THAN, lambda + aHalfWidth);
  cpl_size nsel = cpl_table_count_selected(aPixtable->table);
  cpl_ensure(nsel > 0, CPL_ERROR_DATA_NOT_FOUND, 0.);

  /* now resample all the selected pixels into a spectrum *
   * with hopefully high resolution given by aBinSize     */
  cpl_errorstate state = cpl_errorstate_get();
  cpl_table *spec = muse_resampling_spectrum_iterate(aPixtable, aBinSize,
                                                     aLo, aHi, aIter);
  cpl_table_unselect_all(aPixtable->table);
  if (!cpl_errorstate_is_equal(state)) {
    /* something went wrong with the creation of the spectrum */
    cpl_table_delete(spec);
    cpl_error_set(__func__, cpl_error_get_code());
    return 0.;
  }
  if (flip) {
    /* just change the sign of the data column of the spectrum; it does not *
     * seem to be necessary to move the spectrum above the zero level again */
    cpl_table_multiply_scalar(spec, "data", -1.);
  }

  /* check that the spectrum is contiguous */
  cpl_size nbins = cpl_table_get_nrow(spec);
  /* turn the stat column (in sigma^2) into an error column (in sigma) */
  cpl_table_power_column(spec, "stat", 0.5); /* sqrt() */
  cpl_table_name_column(spec, "stat", "error");
  cpl_table_set_column_unit(spec, "error",
                            cpl_table_get_column_unit(spec, "data"));
#if 0
  char *fn = cpl_sprintf("spec_ifu%02hhu_%p_%.3f.fits",
                         muse_utils_get_ifu(aPixtable->header),
                         (void *)aPixtable, aLambda);
  cpl_msg_debug(__func__, "----> saving spectrum as \"%s\"", fn);
  cpl_table_save(spec, NULL, NULL, fn, CPL_IO_CREATE);
  cpl_free(fn);
#endif
  /* wrap table columns into vectors for the Gaussian fit */
  cpl_vector *pos = cpl_vector_wrap(nbins, cpl_table_get_data_double(spec, "lambda")),
             *val = cpl_vector_wrap(nbins, cpl_table_get_data_double(spec, "data")),
             *err = cpl_vector_wrap(nbins, cpl_table_get_data_double(spec, "error"));
  state = cpl_errorstate_get();
  double xc, xerr = 2 * aHalfWidth, /* default error as large as the window */
         sigma, area, bglevel, mse;
  cpl_matrix *covariance = NULL;
  cpl_error_code rc = cpl_vector_fit_gaussian(pos, NULL, val, err, CPL_FIT_ALL,
                                              &xc, &sigma, &area, &bglevel,
                                              &mse, NULL, &covariance);
  cpl_vector_unwrap(pos);
  cpl_vector_unwrap(val);
  cpl_vector_unwrap(err);
  cpl_table_delete(spec);
  if (rc == CPL_ERROR_CONTINUE) { /* fit didn't converge */
    /* estimate position error as sigma^2/area as CPL docs suggest */
    xerr = sqrt(sigma * sigma / area);
    cpl_errorstate_set(state); /* error handled, reset it */
  } else if (rc == CPL_ERROR_SINGULAR_MATRIX || !covariance) {
    xerr = sqrt(sigma * sigma / area);
    cpl_errorstate_set(state); /* error handled, reset it */
  } else {
    xerr = sqrt(cpl_matrix_get(covariance, 0, 0));
#if 0
    cpl_msg_debug(__func__, "covariance matrix:");
    cpl_matrix_dump(covariance, stdout);
    fflush(stdout);
#endif
  }
  if (aResults && cpl_array_get_type(aResults) == CPL_TYPE_DOUBLE) {
    cpl_array_set_size(aResults, 4);
    cpl_array_set_double(aResults, 0, xc);
    cpl_array_set_double(aResults, 1, sigma);
    cpl_array_set_double(aResults, 2, area);
    cpl_array_set_double(aResults, 3, bglevel);
  } /* valid aResults array */
  if (aErrors && cpl_array_get_type(aErrors) == CPL_TYPE_DOUBLE) {
    cpl_array_set_size(aErrors, 4);
    cpl_array_set_double(aErrors, 0, xerr);
    if (rc != CPL_ERROR_NONE || !covariance) {
      cpl_array_fill_window_invalid(aErrors, 1, 3);
    } else {
      cpl_array_set_double(aErrors, 1, sqrt(cpl_matrix_get(covariance, 1, 1)));
      cpl_array_set_double(aErrors, 2, sqrt(cpl_matrix_get(covariance, 2, 2)));
      cpl_array_set_double(aErrors, 3, sqrt(cpl_matrix_get(covariance, 3, 3)));
    } /* else */
  } /* valid aErrors array */
  cpl_matrix_delete(covariance);
  cpl_msg_debug(__func__, "Gaussian fit (%s): %f +/- %f Angstrom, %f, %f, %f "
                "(RMS %f)", rc != CPL_ERROR_NONE ? cpl_error_get_message() : "",
                xc, xerr, bglevel, area, sigma, sqrt(mse));
  return xc;
} /* muse_utils_pixtable_fit_line_gaussian() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief  Smooth a spectrum in a table with a running mean or median.
  @param  aSpec        the CPL table to be smoothed
  @param  aLambdaCol   column containing wavelengths
  @param  aFluxCol     column containing fluxes
  @param  aErrorCol    column containing flux errors (optional, can be NULL)
  @param  aHalfwidth   the smoothing halfwidth in Angstrom
  @param  aLambdaMin   minimum wavelength to work with
  @param  aLambdaMax   maximum wavelength to work with
  @param  aGapMin      minimum gap wavelength (> aGapMax for no gap)
  @param  aGapMax      maximum gap wavelength (< aGapMin for no gap)
  @param  aAverage     use sliding average instead of sliding median

  Compute a sliding median filter with 2 x aHalfwidth size over the data in
  aSpec, using the given column names for the data. Set the output
  errorbars as the median absolute deviation in the region of the filter width
  at each point, or the median of all input errorbars, whatever is larger.

  If aGapMin is smaller than aGapMax and both are within the wavelength range,
  this region is excluded from smoothing.

  If aLambdaMin and/or aLambdaMax are given such that only a subset of the table
  is smoothed, the smoothing is done symmetrically around each wavelength.

  aSpec is expected to contain the given table columns.

  @error{set and return CPL_ERROR_NULL_INPUT,
         aSpec\, aFluxCol\, or aLambdaCol are NULL}
  @error{set and return CPL_ERROR_ILLEGAL_INPUT,
         one of the given column names was not present in the table}
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
muse_utils_spectrum_smooth_running(cpl_table *aSpec, const char *aLambdaCol,
                                   const char *aFluxCol, const char *aErrorCol,
                                   double aHalfwidth,
                                   double aLambdaMin, double aLambdaMax,
                                   double aGapMin, double aGapMax,
                                   cpl_boolean aAverage)
{
  cpl_ensure_code(aSpec && aLambdaCol && aFluxCol, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(cpl_table_has_column(aSpec, aLambdaCol) &&
                  cpl_table_has_column(aSpec, aFluxCol) &&
                  (aErrorCol ? cpl_table_has_column(aSpec, aErrorCol) : CPL_TRUE),
                  CPL_ERROR_ILLEGAL_INPUT);

  cpl_msg_debug(__func__, "gap (%.3f..%.3f) wavelength range (%.3f..%.3f)",
                aGapMin, aGapMax, aLambdaMin, aLambdaMax);
  /* if a gap was given, call this function recursively, *
   * once for each spectral range outside the gap        */
  if (aGapMin < aGapMax) { /* valid gap */
    cpl_errorstate state = cpl_errorstate_get();
    if (aGapMin > aLambdaMin && aGapMax < aLambdaMax) {
      /* standard case for NaD notch: gap within the wavelength range *
       * need two calls, separately for both full ranges              */
      muse_utils_spectrum_smooth_running(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                         aHalfwidth, aLambdaMin, aGapMin,
                                         0.1, -0.1, aAverage);
      muse_utils_spectrum_smooth_running(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                         aHalfwidth, aGapMax, aLambdaMax,
                                         0.1, -0.1, aAverage);
#if 0 /* other cases that currently do not happen */
    } else if (aGapMin < aLambdaMin && aGapMax > aLambdaMax) {
      /* bad case: only NaD range, no smoothing possible */
      cpl_msg_warning(__func__, "No smoothing, gap (%.3f..%.3f) is larger than "
                      "wavelength range (%.3f..%.3f).", aGapMin, aGapMax,
                      aLambdaMin, aLambdaMax);
    } else if (aGapMin <= aLambdaMin && aGapMax < aLambdaMax) {
      /* 3rd case: gap at lower limit of wavelength range, only one call */
      muse_utils_spectrum_smooth_running(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                         aHalfwidth, aGapMax, aLambdaMax,
                                         0.1, -0.1, aAverage);
    } else if (aGapMin > aLambdaMin && aGapMax >= aLambdaMax) {
      /* 4th case: gap at upper limit of wavelength range , only one call */
      muse_utils_spectrum_smooth_running(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                         aHalfwidth, aLambdaMin, aGapMin,
                                         0.1, -0.1, aAverage);
#endif
    }
    return cpl_errorstate_is_equal(state) ? CPL_ERROR_NONE
                                          : cpl_error_get_code();
  } /* if valid gap */

  /* duplicate the input columns, to not disturb the smoothing while running */
  cpl_table_duplicate_column(aSpec, "ftmp", aSpec, aFluxCol);
  if (aErrorCol) {
    cpl_table_duplicate_column(aSpec, "etmp", aSpec, aErrorCol);
  }

  /* select the rows which to use for the smoothing */
  cpl_table_select_all(aSpec);
  cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_LESS_THAN, aLambdaMin);
  cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_GREATER_THAN, aLambdaMax);
  /* unselect entries with invalid flux column */
  int i, n = cpl_table_get_nrow(aSpec);
  for (i = 0; i < n; i++) {
    if (!cpl_table_is_valid(aSpec, aFluxCol, i)) {
      cpl_table_unselect_row(aSpec, i);
    }
  }
  cpl_boolean sym = cpl_table_count_selected(aSpec) < cpl_table_get_nrow(aSpec);
  cpl_msg_debug(__func__, "%s smoothing +/- %.3f Angstrom between %.3f and %.3f"
                " Angstrom", sym ? "symmetrical" : "", aHalfwidth, aLambdaMin,
                aLambdaMax);

  /* sliding median to get the values, and its median deviation for the error */
  for (i = 0; i < n; i++) {
    if (!cpl_table_is_selected(aSpec, i)) {
      continue;
    }
    double lambda = cpl_table_get_double(aSpec, aLambdaCol, i, NULL);
    int j = i, j1 = i, j2 = i;
    /* search for the range */
    while (--j > 0 && cpl_table_is_selected(aSpec, j) &&
           lambda - cpl_table_get_double(aSpec, aLambdaCol, j, NULL) <= aHalfwidth) {
      j1 = j;
    }
    j = i;
    while (++j < n && cpl_table_is_selected(aSpec, j) &&
           cpl_table_get_double(aSpec, aLambdaCol, j, NULL) - lambda <= aHalfwidth) {
      j2 = j;
    }
    if (sym) { /* adjust ranges to the smaller one for symmetrical smoothing */
      int jd1 = i - j1,
          jd2 = j2 - i;
      if (jd1 < jd2) {
        j2 = i + jd1;
      } else {
        j1 = i - jd2;
      }
    } /* if sym */
    double *sens = cpl_table_get_data_double(aSpec, "ftmp"),
           *serr = NULL;
    cpl_vector *v = cpl_vector_wrap(j2 - j1 + 1, sens + j1),
               *ve = NULL;
    if (aErrorCol) {
      serr = cpl_table_get_data_double(aSpec, "etmp");
      ve = cpl_vector_wrap(j2 - j1 + 1, serr + j1);
    }
    if (aAverage) {
      /* sliding average, use real stdev for >1 points */
      double mean = cpl_vector_get_mean(v),
             stdev = j2 == j1 ? 0. : cpl_vector_get_stdev(v),
             error = 0.;
      cpl_table_set_double(aSpec, aFluxCol, i, mean);
      if (aErrorCol) {
        double rerr = cpl_table_get_double(aSpec, aErrorCol, i, NULL);
        error = sqrt(rerr*rerr + stdev*stdev);
        cpl_table_set_double(aSpec, aErrorCol, i, error);
      }
#if 0
      cpl_msg_debug(__func__, "%d %.3f %d...%d --> %f +/- %f", i, lambda, j1, j2,
                    mean, error);
#endif
    } else {
      /* sliding median */
      double median = cpl_vector_get_median_const(v),
             mdev = muse_cplvector_get_adev_const(v, median);
      if (aErrorCol) {
        double mederr = cpl_vector_get_median_const(ve);
        if (j2 == j1) { /* for single points, copy the error */
          mdev = cpl_table_get_double(aSpec, "etmp", j1, NULL);
        }
        if (mdev < mederr) {
          mdev = mederr;
        }
        cpl_table_set_double(aSpec, aErrorCol, i, mdev);
      } /* if */
#if 0
      cpl_msg_debug(__func__, "%d %.3f %d...%d --> %f +/- %f", i, lambda, j1, j2,
                    median, mdev);
#endif
      cpl_table_set_double(aSpec, aFluxCol, i, median);
    } /* else */
    cpl_vector_unwrap(v);
    cpl_vector_unwrap(ve);
  } /* for i (all aSpec rows) */

  /* erase the extra columns again */
  cpl_table_erase_column(aSpec, "ftmp");
  if (aErrorCol) {
    cpl_table_erase_column(aSpec, "etmp");
  }
  return CPL_ERROR_NONE;
} /* muse_utils_spectrum_smooth_running() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief  Collect spectral points for smoothing from within a wavelength range.
  @param  aTable    the table to use
  @param  aLambda   the central wavelength
  @param  aLDist    the half-width of the wavelength range
  @param  aPos      the (output) matrix of positions
  @param  aVal      the (output) vector of values
  @param  aErr      the (output) vector of errors

  This is used to fill matrix and vectors to carry out a polynomial fit.
  aTable is expected to contain the columns "lambda", "ftmp", and (optionally!)
  "etmp".
 */
/*----------------------------------------------------------------------------*/
static unsigned int
muse_utils_spectrum_smooth_collect_points(const cpl_table *aTable,
                                          double aLambda, double aLDist,
                                          cpl_matrix *aPos, cpl_vector *aVal,
                                          cpl_vector *aErr)
{
  unsigned int np = 0; /* counter for number of transfered points */
  int irow, nrow = cpl_table_get_nrow(aTable),
      nsel = cpl_table_count_selected(aTable);
  cpl_boolean haserrors = cpl_table_has_column(aTable, "etmp");
  /* use the selected rows, if there are some (but not all are selected) */
  cpl_boolean selected = nsel > 0 && nsel != nrow;
  for (irow = 0; irow < nrow; irow++) {
    if (selected && !cpl_table_is_selected(aTable, irow)) {
      continue; /* ignore this row if it's not selected but we want those */
    }
    double lambda = cpl_table_get(aTable, "lambda", irow, NULL);
    if (lambda < aLambda - aLDist || lambda > aLambda + aLDist) {
      continue;
    }
    cpl_matrix_set(aPos, 0, np, lambda);
    int err;
    cpl_vector_set(aVal, np, cpl_table_get(aTable, "ftmp", irow, &err));
    if (err) {
      continue;
    }
    if (haserrors) {
      cpl_vector_set(aErr, np, cpl_table_get(aTable, "etmp", irow, NULL));
    }
    np++;
  } /* for irow */
  cpl_matrix_set_size(aPos, 1, np);
  cpl_vector_set_size(aVal, np);
  if (haserrors) {
    cpl_vector_set_size(aErr, np);
  }
  return np;
} /* muse_utils_spectrum_smooth_collect_points() */

/*----------------------------------------------------------------------------*/
/**
  @private
  @brief  Smooth a spectrum in a table with piecewise cubic polynomials.
  @param  aSpec        the CPL table to be smoothed
  @param  aLambdaCol   column containing wavelengths
  @param  aFluxCol     column containing fluxes
  @param  aErrorCol    column containing flux errors (optional, can be NULL)
  @param  aLambdaMin   minimum wavelength
  @param  aLambdaMax   maximum wavelength
  @param  aGapMin      minimum gap wavelength (> aGapMax for no gap)
  @param  aGapMax      maximum gap wavelength (< aGapMin for no gap)
  @param  aDStep       half-width around each wavelength position
  @param  aRSigma      rejection sigma level to use, in case an iterative fit is
                       required

  This smooths the spectrum in the table aSpec aResp using cubic piecewise
  polynomials, that are computed at each wavelength point of the table. Each
  polynomial is fit iteratively, if points more deviant than aRSigma x RMS are
  found.

  If aGapMin is smaller than aGapMax and both are within the wavelength range,
  this region is excluded from smoothing.

  aSpec is expected to contain the given table columns.

  @error{set and return CPL_ERROR_NULL_INPUT,
         aSpec\, aFluxCol\, or aLambdaCol are NULL}
  @error{set and return CPL_ERROR_ILLEGAL_INPUT,
         one of the given column names was not present in the table}
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
muse_utils_spectrum_smooth_ppoly(cpl_table *aSpec, const char *aLambdaCol,
                                 const char *aFluxCol, const char *aErrorCol,
                                 double aLambdaMin, double aLambdaMax,
                                 double aGapMin, double aGapMax, double aDStep,
                                 float aRSigma)
{
  cpl_ensure_code(aSpec && aLambdaCol && aFluxCol, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(cpl_table_has_column(aSpec, aLambdaCol) &&
                  cpl_table_has_column(aSpec, aFluxCol) &&
                  (aErrorCol ? cpl_table_has_column(aSpec, aErrorCol) : CPL_TRUE),
                  CPL_ERROR_ILLEGAL_INPUT);

  cpl_msg_debug(__func__, "gap (%.3f..%.3f) wavelength range (%.3f..%.3f)",
                aGapMin, aGapMax, aLambdaMin, aLambdaMax);
  /* if a gap was given, call this function recursively, *
   * once for each spectral range outside the gap        */
  if (aGapMin < aGapMax) { /* valid gap */
    cpl_errorstate state = cpl_errorstate_get();
    if (aGapMin > aLambdaMin && aGapMax < aLambdaMax) {
      /* standard case for NaD notch: gap within the wavelength range *
       * need two calls, separately for both full ranges              */
      muse_utils_spectrum_smooth_ppoly(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                       aLambdaMin, aGapMin, 0.1, -0.1, aDStep,
                                       aRSigma);
      muse_utils_spectrum_smooth_ppoly(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                       aGapMax, aLambdaMax, 0.1, -0.1, aDStep,
                                       aRSigma);
#if 0 /* other cases that currently do not happen */
    } else if (aGapMin < aLambdaMin && aGapMax > aLambdaMax) {
      /* bad case: only NaD range, no smoothing possible */
      cpl_msg_warning(__func__, "No piecewise polynomial smoothing, gap (%.3f.."
                      "%.3f) is larger than wavelength range (%.3f..%.3f).",
                      aGapMin, aGapMax, aLambdaMin, aLambdaMax);
    } else if (aGapMin <= aLambdaMin && aGapMax < aLambdaMax) {
      /* 3rd case: gap at lower limit of wavelength range, only one call */
      muse_utils_spectrum_smooth_ppoly(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                      aGapMax, aLambdaMax, 0.1, -0.1, aDStep,
                                      aRSigma);
    } else if (aGapMin > aLambdaMin && aGapMax >= aLambdaMax) {
      /* 4th case: gap at upper limit of wavelength range , only one call */
      muse_utils_spectrum_smooth_ppoly(aSpec, aLambdaCol, aFluxCol, aErrorCol,
                                      aLambdaMin, aGapMin, 0.1, -0.1, aDStep,
                                      aRSigma);
#endif
    }
    return cpl_errorstate_is_equal(state) ? CPL_ERROR_NONE
                                          : cpl_error_get_code();
  } /* if valid gap */

  /* duplicate the input columns, to not disturb the smoothing while running */
  cpl_table_duplicate_column(aSpec, "ftmp", aSpec, aFluxCol);
  if (aErrorCol) {
    cpl_table_duplicate_column(aSpec, "etmp", aSpec, aErrorCol);
  }

  /* select the rows which to use for the smoothing */
  cpl_table_select_all(aSpec);
  cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_LESS_THAN,
                                aLambdaMin);
  cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_GREATER_THAN,
                                aLambdaMax);
  /* unselect entries with invalid flux column */
  int i, n = cpl_table_get_nrow(aSpec);
  for (i = 0; i < n; i++) {
    if (!cpl_table_is_valid(aSpec, aFluxCol, i)) {
      cpl_table_unselect_row(aSpec, i);
    }
  }

  /* variables to keep track of jumps that need to be fixed afterwards */
  unsigned int npold = 0, njumps = 0;
  double ldistold = -1,
         lambdaold = -1;
  cpl_array *jumppos = cpl_array_new(0, CPL_TYPE_DOUBLE),
            *jumplen = cpl_array_new(0, CPL_TYPE_DOUBLE);

  /* compute the piecewise cubic polynomial for the data around each input datapoint */
  int irow, nrow = cpl_table_get_nrow(aSpec);
  for (irow = 0; irow < nrow; irow++) {
    if (!cpl_table_is_selected(aSpec, irow)) {
      continue;
    }

    double lambda = cpl_table_get(aSpec, aLambdaCol, irow, NULL);
    /* set up breakpoints every aDStep Angstrom */
    double ldist = aDStep;
    /* collect all data ldist / 2 below and ldist / 2 above the knot wavelength */
    cpl_matrix *pos = cpl_matrix_new(1, nrow); /* start with far too long ones */
    cpl_vector *val = cpl_vector_new(nrow),
               *err = NULL;
    if (aErrorCol) {
      err = cpl_vector_new(nrow);
    }
    /* this call uses "ftmp" and possibly "etmp": */
    unsigned int np = muse_utils_spectrum_smooth_collect_points(aSpec, lambda,
                                                                ldist, pos, val,
                                                                err);
    if (ldistold < 0) {
      npold = np;
      ldistold = ldist;
      lambdaold = lambda;
    }
#if 0
    printf("%f Angstrom %u points (%u, %.3f, %f):\n", lambda, np, npold,
           (double)np / npold - 1., cpl_vector_get_mean(val));
    cpl_matrix_dump(pos, stdout);
    fflush(stdout);
#endif
    /* the number of points changed more than 10% */
    if (np > 10 && fabs((double)np / npold - 1.) > 0.1) {
      cpl_msg_debug(__func__, "possible jump, changed at lambda %.3f (%u -> %u, "
                    "%.3f -> %.3f)", lambda, npold, np, ldistold, ldist);
      cpl_array_set_size(jumppos, ++njumps);
      cpl_array_set_size(jumplen, njumps);
      cpl_array_set_double(jumppos, njumps - 1, (lambdaold + lambda) / 2.);
      cpl_array_set_double(jumplen, njumps - 1, (ldistold + ldist) / 2.);
    }
    /* we want to simulate a cubic spline, i.e. use order 3, *
     * but we have to do with the number of points we got    */
    unsigned int order = np > 3 ? 3 : np - 1;
    double mse;
    /* fit the polynomial, but without rejection, i.e. use high rejection sigma */
    cpl_polynomial *poly = muse_utils_iterate_fit_polynomial(pos, val, err,
                                                             NULL, order, aRSigma,
                                                             &mse, NULL);
#if 0
    if (fabs(lambda - 4861.3) < 1.) {
      cpl_vector *res = cpl_vector_new(cpl_vector_get_size(val));
      cpl_vector_fill_polynomial_fit_residual(res, val, NULL, poly, pos, NULL);
      double rms = sqrt(cpl_vector_product(res, res) / cpl_vector_get_size(res));
      cpl_msg_debug(__func__, "lambda %f rms %f (%u/%d points)", lambda, rms,
                    (unsigned)cpl_vector_get_size(val), np);
      cpl_polynomial_dump(poly, stdout);
      //cpl_plot_vector(NULL, NULL, NULL, val);
      //cpl_plot_vector(NULL, NULL, NULL, res);
      unsigned int ii;
      for (ii = 0; ii < np; ii++) {
        printf("%.1f %f %f %f\n", cpl_matrix_get(pos, 0, ii),
               cpl_vector_get(val, ii), cpl_vector_get(err, ii),
               cpl_vector_get(res, ii));
      } /* for ii */
      fflush(stdout);
      cpl_vector_delete(res);
    }
#endif
    cpl_matrix_delete(pos);
    cpl_vector_delete(val);
    cpl_vector_delete(err);
    double resp = cpl_polynomial_eval_1d(poly, lambda, NULL);
    cpl_polynomial_delete(poly);
    cpl_table_set(aSpec, aLambdaCol, irow, lambda);
    cpl_table_set(aSpec, aFluxCol, irow, resp);
    if (aErrorCol) {
      double error = cpl_table_get(aSpec, "etmp", irow, NULL);
      cpl_table_set(aSpec, aErrorCol, irow, sqrt(mse + error*error));
    }

    npold = np;
    ldistold = ldist;
    lambdaold = lambda;
  } /* for i (all rows) */

  /* erase the extra columns again */
  cpl_table_erase_column(aSpec, "ftmp");
  if (aErrorCol) {
    cpl_table_erase_column(aSpec, "etmp");
  }

#if 0
  printf("jumppos (%u):\n", njumps);
  cpl_array_dump(jumppos, 0, 10000, stdout);
  printf("jumplen:\n");
  cpl_array_dump(jumplen, 0, 10000, stdout);
  fflush(stdout);
#endif

  /* this needs median smoothing afterwards as well *
   * to remove the jumps where the length changed   */
  unsigned int iarr;
  for (iarr = 0; iarr < njumps; iarr++) {
    double lambda = cpl_array_get_double(jumppos, iarr, NULL),
           ldist = cpl_array_get_double(jumplen, iarr, NULL) / 2;
    /* check that the step in the +/- 5 Angstrom region actually worth worrying about */
    cpl_table_select_all(aSpec);
    cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_LESS_THAN,
                                  lambda - 5.);
    cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_GREATER_THAN,
                                  lambda + 5.);
    int nsel = cpl_table_count_selected(aSpec);
    if (nsel <= 1) {
      cpl_msg_debug(__func__, "Only %d points near jump around %.1f Angstrom, "
                    "not doing anything", nsel, lambda);
      continue;
    }
    cpl_table *xresp = cpl_table_extract_selected(aSpec);
    double stdev = cpl_table_get_column_stdev(xresp, "response");
#if 0
    cpl_table_dump(xresp, 0, nsel, stdout);
    fflush(stdout);
#endif
    cpl_table_delete(xresp);
    if (stdev < 0.001) {
      cpl_msg_debug(__func__, "%d points near jump around %.1f Angstrom, stdev "
                    "only %f, not doing anything", nsel, lambda, stdev);
      continue;
    }
    cpl_msg_debug(__func__, "%d points near jump around %.1f Angstrom, stdev "
                  "%f, erasing the region", nsel, lambda, stdev);

    /* erase the affected part of the response, linear       *
     * interpolation when applying will take care of the gap */
    cpl_table_select_all(aSpec);
    cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_LESS_THAN,
                                  lambda - ldist);
    cpl_table_and_selected_double(aSpec, aLambdaCol, CPL_NOT_GREATER_THAN,
                                  lambda + ldist);
    cpl_table_erase_selected(aSpec);
  } /* for iarr (all jump points) */
  cpl_array_delete(jumppos);
  cpl_array_delete(jumplen);
  return CPL_ERROR_NONE;
} /* muse_utils_spectrum_smooth_ppoly() */

/*----------------------------------------------------------------------------*/
/**
  @brief  Smooth a spectrum in a table.
  @param  aSpec       a MUSE table containing the spectrum to smooth.
  @param  aSmooth     the smoothing type to use
  @return CPL_ERROR_NONE on success, another CPL error code on failure

  One can choose to either smooth a response curve (with format @ref
  muse_flux_responsetable_def, and columns "lambda", "response", and "resperr")
  or a flat-field spectrum (with columns MUSE_PIXTABLE_FFLAMBDA and
  MUSE_PIXTABLE_FFDATA).

  The smoothing options are given by the @ref muse_spectrum_smooth_type.

  To reject outliers, the reponse function is smoothed using either a median
  with 15 Angstrom halfwidth or a piecewise cubic polynomial followed by a
  sliding average of 15 Angstrom halfwidth.

  For the median filter, the output errorbars are taken as the median absolute
  deviation in the same region, or the median of all input errorbars, whatever
  is larger. For the piecewise polynomial, the error of the fit and the standard
  deviation of the averaging region are propagated onto the original error of
  each point.

  @error{set and return CPL_ERROR_NULL_INPUT,
         aSpec or one of its components is NULL}
  @error{return CPL_ERROR_UNSUPPORTED_MODE,
         did not find a supported spectral table format}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_utils_spectrum_smooth(muse_table *aSpec, muse_spectrum_smooth_type aSmooth)
{
  cpl_ensure_code(aSpec && aSpec->table && aSpec->header, CPL_ERROR_NULL_INPUT);

  /* start by setting up names for output and the expected table columns */
  const char *specstring = "",
             *speclam = NULL,
             *specdata = NULL,
             *specerr = NULL;
  if (cpl_table_has_column(aSpec->table, "lambda") &&
      cpl_table_has_column(aSpec->table, "response")) {
    specstring = "response curve";
    speclam = "lambda";
    specdata = "response";
    if (cpl_table_has_column(aSpec->table, "resperr")) {
      specerr = "resperr";
    }
  } else if (cpl_table_has_column(aSpec->table, MUSE_PIXTABLE_FFLAMBDA) &&
             cpl_table_has_column(aSpec->table, MUSE_PIXTABLE_FFDATA)) {
    specstring = "flat-field spectrum";
    speclam = MUSE_PIXTABLE_FFLAMBDA;
    specdata = MUSE_PIXTABLE_FFDATA;
    /* specerr stays NULL */
  } else {
    specstring = "unknown spectrum";
  }
  if (!speclam || !specdata) {
    cpl_msg_warning(__func__, "Cannot smooth %s!", specstring);
    return CPL_ERROR_UNSUPPORTED_MODE;
  }
  if (aSmooth == MUSE_SPECTRUM_SMOOTH_NONE) {
    cpl_msg_warning(__func__, "Not smoothing the %s at all!", specstring);
    return CPL_ERROR_NONE;
  }

  /* Check, if we need to exclude the blue part. In nominal non-AO mode as *
   * well as in nominal AO mode, with flat-field spectrum correction, the  *
   * sensitivity has a very narrow but real kink where the blue cutoff     *
   * filter starts. We need to stop smoothing just redward of that.        *
   * Also set up the smoothing gap here, if needed to skip the NaD range.  *
   * We need to handle the gap as well for the case of a flat-field        *
   * which will always have the deep drop around NaD.                      */
  double blueend = kMuseLambdaMinX,
         gapmin = 0.1, gapmax = -0.1; /* setup no gap for a start */
  if (!strncmp(specstring, "flat-field", 10) ||
      (!strncmp(specstring, "response", 8) &&
       cpl_propertylist_has(aSpec->header, MUSE_HDR_FLUX_FFCORR))) {
    muse_ins_mode mode = muse_pfits_get_mode(aSpec->header);
    if (mode == MUSE_MODE_WFM_NONAO_N) {
      /* normal blue cutoff filter, but no notch gap */
      blueend = kMuseNominalCutoffKink;
    } else if (mode == MUSE_MODE_WFM_AO_N) {
      /* AO notch filter, with different blue cutoff and a gap */
      blueend = kMuseAOCutoffKink;
      gapmin = kMuseNa2LambdaMin;
      gapmax = kMuseNa2LambdaMax;
    } else if (mode == MUSE_MODE_WFM_AO_E) {
      /* AO-E notch filter, no blue cutoff but a different notch gap */
      gapmin = kMuseNaLambdaMin;
      gapmax = kMuseNaLambdaMax;
    } else if (mode == MUSE_MODE_NFM_AO_N) {
      /* GALACSI notch filter plus Blue-IR MUSE filter, with different *
       * blue cutoff and a gap                                         */
      blueend = kMuseNFMCutoffKink;
      if (!strncmp(specstring, "response", 8)) {
        /* since NFM lamp flats have no NaD filter in them,   *
         * only set up a gap for response curves in this mode */
        gapmin = kMuseNaGLambdaMin;
        gapmax = kMuseNaGLambdaMax;
      }
    } /* else if */
  } /* if */

  /* now smooth the response */
  if (aSmooth == MUSE_SPECTRUM_SMOOTH_MEDIAN) {
    cpl_msg_info(__func__, "Smoothing %s with median filter", specstring);
    /* use a sliding median over +/- 15 Angstrom width */
    muse_utils_spectrum_smooth_running(aSpec->table, speclam, specdata, specerr,
                                       15., blueend, kMuseLambdaMaxX,
                                       gapmin, gapmax, CPL_FALSE);
  } else { /* aSmooth == MUSE_SPECTRUM_SMOOTH_PPOLY */
    cpl_msg_info(__func__, "Smoothing %s with piecewise polynomial, plus "
                 "running average", specstring);
    /* use a piecewise polynomial, i.e. a simple, non-contiguous spline */
    muse_utils_spectrum_smooth_ppoly(aSpec->table, speclam, specdata, specerr,
                                     blueend, kMuseLambdaMaxX, gapmin, gapmax,
                                     150., 3.);
    /* this usually needs extra smoothing afterwards with a sliding average */
    muse_utils_spectrum_smooth_running(aSpec->table, speclam, specdata, specerr,
                                       15., blueend, kMuseLambdaMaxX, gapmin,
                                       gapmax, CPL_TRUE);
  } /* else */

  return CPL_ERROR_NONE;
} /* muse_utils_spectrum_smooth() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Copy a modified header keyword from one header to another.
  @param    aH1        input header
  @param    aH2        output header
  @param    aKeyword   header keyword to copy
  @param    aString    string to be used for the modification
  @return   CPL_ERROR_NONE on success or another CPL error on failure.

  The keyword is modified by appending " (aString)" to it.

  @error{return CPL_ERROR_NULL_INPUT, one of the input arguments is NULL}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         the string in the input header cannot be read}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_utils_copy_modified_header(cpl_propertylist *aH1, cpl_propertylist *aH2,
                                const char *aKeyword, const char *aString)
{
  cpl_ensure_code(aH1 && aH2 && aKeyword && aString, CPL_ERROR_NULL_INPUT);
  const char *hin = cpl_propertylist_get_string(aH1, aKeyword);
  cpl_ensure_code(hin, CPL_ERROR_ILLEGAL_INPUT);
  char *hstring = cpl_sprintf("%s (%s)", hin, aString);
  cpl_error_code rc = cpl_propertylist_update_string(aH2, aKeyword, hstring);
  cpl_free(hstring);

  return rc;
} /* muse_utils_copy_modified_header() */

/*---------------------------------------------------------------------------*/
/**
  @brief  Set HDU headers for the ESO FITS data format.
  @param  aHeader    the FITS headers to modify
  @param  aClass2    type of class to set, i.e. contents of HDUCLAS2
  @param  aExtData   extension name for the data extension
  @param  aExtDQ     extension name for the bad pixel extension (or NULL)
  @param  aExtStat   extension name for the variance extension (or NULL)
  @return CPL_ERROR_NONE on success, another CPL error code on failure.

  This function adds special FITS headers to support the ESO format. The
  information was taken from v2.5.1 (dated Feb. 15th, 2012) of the document
  "FITS format description for pipeline products with data, error and data
  quality information" and further info by Martin Kuemmel and Harald Kuntschner.

  This function tries to set these header keywords directly after EXTNAME, if
  available, otherwise at the end of the input header.

  @qa This function is used by @ref muse_image_save() and @ref
      muse_datacube_save(). Tests of their functionality include tests of the
      headers that are created using this function.

  @error{return CPL_ERROR_NULL_INPUT, aHeader\, aClass2\, or aExtData are NULL}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         aClass2 is neither "DATA"\, "ERROR"\, nor "QUALITY"}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_utils_set_hduclass(cpl_propertylist *aHeader, const char *aClass2,
                        const char *aExtData, const char *aExtDQ,
                        const char *aExtStat)
{
  cpl_ensure_code(aHeader && aClass2 && aExtData, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(!strcmp(aClass2, "DATA") || !strcmp(aClass2, "ERROR") ||
                 !strcmp(aClass2, "QUALITY"), CPL_ERROR_ILLEGAL_INPUT);

  /* first clean up existing entries */
  cpl_propertylist_erase_regexp(aHeader, ESO_HDU_HEADERS_REGEXP, 0);

  if (cpl_propertylist_has(aHeader, "EXTNAME")) {
    cpl_propertylist_insert_after_string(aHeader, "EXTNAME", "HDUCLASS", "ESO");
  } else {
    cpl_propertylist_append_string(aHeader, "HDUCLASS", "ESO");
  }
  cpl_propertylist_set_comment(aHeader, "HDUCLASS", "class name (ESO format)");
  cpl_propertylist_insert_after_string(aHeader, "HDUCLASS", "HDUDOC", "DICD");
  cpl_propertylist_set_comment(aHeader, "HDUDOC", "document with class description");
  cpl_propertylist_insert_after_string(aHeader, "HDUDOC", "HDUVERS",
                                       "DICD version 6");
  cpl_propertylist_set_comment(aHeader, "HDUVERS",
                               "version number (according to spec v2.5.1)");
  cpl_propertylist_insert_after_string(aHeader, "HDUVERS", "HDUCLAS1", "IMAGE");
  cpl_propertylist_set_comment(aHeader, "HDUCLAS1", "Image data format");
  cpl_propertylist_insert_after_string(aHeader, "HDUCLAS1", "HDUCLAS2", aClass2);
  if (!strcmp(aClass2, "DATA")) { /* this is the STAT / variance extension */
    cpl_propertylist_set_comment(aHeader, "HDUCLAS2",
                                 "this extension contains the data itself");
    /* no HDUCLAS3 for the data itself */
    if (aExtDQ) {
      cpl_propertylist_insert_after_string(aHeader, "HDUCLAS2", "QUALDATA", aExtDQ);
    }
    if (aExtStat) { /* do this second, so that it goes first, if given */
      cpl_propertylist_insert_after_string(aHeader, "HDUCLAS2", "ERRDATA", aExtStat);
    }
  } else if (!strcmp(aClass2, "ERROR")) { /* this is the STAT / variance extension */
    cpl_propertylist_set_comment(aHeader, "HDUCLAS2", "this extension contains variance");
    cpl_propertylist_insert_after_string(aHeader, "HDUCLAS2", "HDUCLAS3", "MSE");
    cpl_propertylist_set_comment(aHeader, "HDUCLAS3",
                                 "the extension contains variances (sigma**2)");
    cpl_propertylist_insert_after_string(aHeader, "HDUCLAS3", "SCIDATA", aExtData);
    if (aExtDQ) {
      cpl_propertylist_insert_after_string(aHeader, "SCIDATA", "QUALDATA", aExtDQ);
    }
  } else { /* "QUALITY", this is the DQ / bad pixel extension */
    cpl_propertylist_set_comment(aHeader, "HDUCLAS2",
                                 "this extension contains bad pixel mask");
    cpl_propertylist_insert_after_string(aHeader, "HDUCLAS2", "HDUCLAS3", "FLAG32BIT");
    cpl_propertylist_set_comment(aHeader, "HDUCLAS3", "extension contains 32bit"
                                 " Euro3D bad pixel information");
    /* all non-zero values in the bad pixel mask are "bad" */
    cpl_propertylist_insert_after_long(aHeader, "HDUCLAS3", "QUALMASK", 0xFFFFFFFF);
    cpl_propertylist_set_comment(aHeader, "QUALMASK", "all non-zero values are bad");
    cpl_propertylist_insert_after_string(aHeader, "QUALMASK", "SCIDATA", aExtData);
    if (aExtStat) {
      cpl_propertylist_insert_after_string(aHeader, "SCIDATA", "ERRDATA", aExtStat);
    }
  }

  if (cpl_propertylist_has(aHeader, "SCIDATA")) {
    cpl_propertylist_set_comment(aHeader, "SCIDATA", "pointer to the data extension");
  }
  if (cpl_propertylist_has(aHeader, "ERRDATA")) {
    cpl_propertylist_set_comment(aHeader, "ERRDATA", "pointer to the variance extension");
  }
  if (cpl_propertylist_has(aHeader, "QUALDATA")) {
    cpl_propertylist_set_comment(aHeader, "QUALDATA",
                                 "pointer to the bad pixel mask extension");
  }

  return CPL_ERROR_NONE;
} /* muse_utils_set_hduclass() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Display the current memory usage of the given program.
  @param    aMarker    marker string to use in the debug output
  @note This function prints CPL memory info to stderr and other data to
        stdout!
  @note This function only outputs anything if an executable name was given
        using the environment variable MUSE_DEBUG_MEMORY_PROGRAM.

  This function uses the cpl_memory_dump() function to list pointers allocated
  through CPL and the Unix ps command to display system information about the
  process.
 */
/*----------------------------------------------------------------------------*/
void
muse_utils_memory_dump(const char *aMarker)
{
  char *exe = getenv("MUSE_DEBUG_MEMORY_PROGRAM");
  if (!exe) {
    return;
  }

  printf("=== %s ===\n", aMarker);
  fflush(stdout);
  char command[1000];
  if (strlen(exe) > 0) {
    snprintf(command, 999,
             "ps -C %s -o comm,start_time,pid,tid,pcpu,stat,rss,size,vsize",
             exe);
  } else {
    /* no program name given, generic output */
    snprintf(command, 999,
             "ps -o comm,start_time,pid,tid,pcpu,stat,rss,size,vsize");
  }
  cpl_memory_dump();
  fflush(stderr);
  int rc = system(command);
  UNUSED_ARGUMENT(rc); /* new glibc mandates use of system() return code */
} /* muse_utils_memory_dump() */

/*----------------------------------------------------------------------------*/
/**
  @brief  Compute the convolution of an image with a kernel.
  @param  aImage  The image to convolve.
  @param  aKernel The convolution kernel.
  @return On success the convoluted image is returned, and @c NULL if an
          error occurred.

  The function convolves the input image @em aImage with the convolution
  kernel @em aKernel using an FFT. If @em aImage has an odd number of pixels
  along the y-axis, the last image row is discarded to make it a size even,
  however, the size of @em aImage along the x-axis must be even and an
  error is returned if it is not.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, a parameter is NULL}
  @error{set CPL_ERROR_IVALID_TYPE\, return NULL, aImage is not if type double}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return NULL, invalid image size}

 */
/*----------------------------------------------------------------------------*/
cpl_image *
muse_convolve_image(const cpl_image *aImage, const cpl_matrix *aKernel)
{
  cpl_ensure(aImage && aKernel, CPL_ERROR_NULL_INPUT, NULL);

  cpl_size nx = cpl_image_get_size_x(aImage);
  cpl_size ny = cpl_image_get_size_y(aImage);
  cpl_size nc = cpl_matrix_get_ncol(aKernel);
  cpl_size nr = cpl_matrix_get_nrow(aKernel);

  cpl_ensure(cpl_image_get_type(aImage) == CPL_TYPE_DOUBLE,
             CPL_ERROR_INVALID_TYPE, NULL);
  cpl_ensure(nx % 2 == 0, CPL_ERROR_INCOMPATIBLE_INPUT, NULL);

  cpl_size kstart[2] = {(nx - nc) / 2, (ny - nr) / 2};
  cpl_size kend[2]   = {kstart[0] + nc, kstart[1] + nr};

  cpl_image *kernel  = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
  double *_kernel  = cpl_image_get_data_double(kernel);

  const double *_aKernel = cpl_matrix_get_data_const(aKernel);
  cpl_size iy;
  for (iy = 0; iy < ny; ++iy) {
    cpl_size ix;
    for (ix = 0; ix < nx; ++ix) {
      cpl_size idx = nx * iy + ix;
      cpl_boolean inside = ((ix >= kstart[0]) && (ix < kend[0])) &&
                           ((iy >= kstart[1]) && (iy < kend[1]));
      if (inside) {
        _kernel[idx] = _aKernel[nc * (iy - kstart[1]) + (ix - kstart[0])];
      }
    }
  }

  cpl_size nxhalf = nx / 2 + 1;
  cpl_image *fft_image  = cpl_image_new(nxhalf, ny, CPL_TYPE_DOUBLE_COMPLEX);
  cpl_image *fft_kernel = cpl_image_new(nxhalf, ny, CPL_TYPE_DOUBLE_COMPLEX);

  cpl_error_code status;
  status = cpl_fft_image(fft_image, aImage, CPL_FFT_FORWARD);
  if (status != CPL_ERROR_NONE) {
    cpl_image_delete(fft_kernel);
    cpl_image_delete(fft_image);
    cpl_image_delete(kernel);
    cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                          "FFT of input image failed!");
    return NULL;
  }
  status = cpl_fft_image(fft_kernel, kernel, CPL_FFT_FORWARD);
  if (status != CPL_ERROR_NONE) {
    cpl_image_delete(fft_kernel);
    cpl_image_delete(fft_image);
    cpl_image_delete(kernel);
    cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                          "FFT of convolution kernel failed!");
    return NULL;
  }
  cpl_image_delete(kernel);

  double complex *_fft_image  = cpl_image_get_data_double_complex(fft_image);
  double complex *_fft_kernel = cpl_image_get_data_double_complex(fft_kernel);

  /* Convolve the input image with the kernel. Compute the two FFTs, for      *
   * the image and the kernel and multiply the results.                       *
   *                                                                          *
   * After the forward transform the zero frequency is stored in the 4        *
   * corners of the result image, and not in the center as one may            *
   * expect. In order to get a correct convolution of the image the input     *
   * buffer of the inverse transformation must be rearranged such that        *
   * the zero frequency appears in the center. This is done by multiplying    *
   * all results at odd index positions in the buffer by -1, which            *
   * effectively swaps the quadrants of the frequency plane.                  *
   * see: www.mvkonnik.info/2014/06/fft-ifft-and-why-do-we-need-fftshift.html */

  for (iy = 0; iy < ny; ++iy) {
    cpl_size ix;
    for (ix = 0; ix < nxhalf; ++ix) {
      cpl_size idx = nxhalf * iy + ix;
      int sign = ((ix + iy) % 2) ? -1 : 1;
      _fft_image[idx] *= sign * _fft_kernel[idx] / (nx * ny);
    }
  }
  cpl_image_delete(fft_kernel);

  cpl_image *result = cpl_image_new(nx, ny, CPL_TYPE_DOUBLE);
  status = cpl_fft_image(result, fft_image, CPL_FFT_BACKWARD | CPL_FFT_NOSCALE);
  if (status != CPL_ERROR_NONE) {
    cpl_image_delete(result);
    cpl_image_delete(fft_image);
    cpl_error_set_message(__func__, CPL_ERROR_INCOMPATIBLE_INPUT,
                          "Backward FFT of convolution result failed!");
    return NULL;
  }
  cpl_image_delete(fft_image);

  return result;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Load a FOV image into a MUSE image.
  @param    aFilename  name of the file to load
  @return   a muse_image or NULL on error

  Load the DATA extension containing the FOV image data from the input
  file. Bad pixels may be encoded as NaN values in the image data directly,
  or, if a DQ extension is found in the input file, it is used to replace
  the bad pixels in the FOV image data with NaN values.

  Alternatively, if the DATA extension is an extension with higher-dimensional
  data or no image content (NAXIS != 2), load the first 2D image extension. In
  that case, search for the related STAT and DQ extension by prefixing them with
  the filter name.
 */
/*----------------------------------------------------------------------------*/
muse_image *
muse_fov_load(const char *aFilename)
{
  muse_image *image = muse_image_new();

  image->header = cpl_propertylist_load(aFilename, 0);
  if (!image->header) {
    cpl_error_set_message(__func__, cpl_error_get_code(), "Loading primary FITS "
                          "header of \"%s\" did not succeed", aFilename);
    muse_image_delete(image);
    return NULL;
  }

  /* assuming an IMAGE_FOV input, start by searching for the DATA extension */
  int extension = cpl_fits_find_extension(aFilename, EXTNAME_DATA);
  cpl_propertylist *hdata = cpl_propertylist_load(aFilename, extension);
  while (muse_pfits_get_naxis(hdata, 0) != 2) {
    /* Not an image, this means we were given a cube, possibly *
     * with image extensions; search for the first of those.   */
    const char *extname = muse_pfits_get_extname(hdata);
    if (!extname && cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) {
      cpl_error_reset();
    }
    cpl_msg_debug(__func__, "Skipping extension %d [%s]", extension,
                  extname ? extname : "<no label>");
    cpl_propertylist_delete(hdata);
    extension++;
    hdata = cpl_propertylist_load(aFilename, extension);
  } /* while */
  cpl_msg_debug(__func__, "Taking extension %d [%s]", extension,
                muse_pfits_get_extname(hdata));
  /* keep the extension name, to check if we loaded *
   * from an IMAGE_FOV or from a cube extension     */
  char *extname = cpl_strdup(muse_pfits_get_extname(hdata));
  image->data = cpl_image_load(aFilename, CPL_TYPE_FLOAT, 0, extension);
  if (!image->data) {
    cpl_error_set_message(__func__, MUSE_ERROR_READ_DATA, "Could not load "
                          "extension %s from \"%s\"", extname, aFilename);
    cpl_free(extname);
    cpl_propertylist_delete(hdata);
    muse_image_delete(image);
    return NULL;
  }

  if (cpl_propertylist_has(hdata, "BUNIT")) {
    cpl_propertylist_append_string(image->header, "BUNIT",
                                   muse_pfits_get_bunit(hdata));
    cpl_propertylist_set_comment(image->header, "BUNIT",
                                 cpl_propertylist_get_comment(hdata, "BUNIT"));
  } else {
    cpl_msg_warning(__func__, "No BUNIT given in extension %d [%s] of \"%s\"!",
                    extension, extname, aFilename);
  }

  if (!cpl_propertylist_has(hdata, "CUNIT1") ||
      !cpl_propertylist_has(hdata, "CUNIT2")) {
    cpl_msg_warning(__func__, "No WCS found in extension %d [%s] of \"%s\"!",
                    extension, extname, aFilename);
  }

  /* Merge WCS keywords and ESO hierarchical keywords from the data   *
   * extension into the image header                                  */
  const char *keys = "(^ESO |" MUSE_WCS_KEYS ")";
  cpl_msg_debug(__func__, "Merging header of extension %d [%s] with primary "
                "header: copying keywords matching '%s'", extension, extname,
                keys);
  cpl_error_code ecode =
      cpl_propertylist_copy_property_regexp(image->header, hdata, keys, 0);
  if (ecode == CPL_ERROR_TYPE_MISMATCH) {
    cpl_error_set_message(__func__, CPL_ERROR_TYPE_MISMATCH,
                          "Merging extension header [\"%s\"] into primary "
                          "FITS header failed! Found keyword in both headers "
                          "where types do not match!", extname);
    cpl_free(extname);
    cpl_propertylist_delete(hdata);
    muse_image_delete(image);
    return NULL;
  }
  cpl_propertylist_delete(hdata);

  /* Load STAT extension if it is found and contains data, ignore it *
   * otherwise */
  if (extname && !strncmp(extname, EXTNAME_DATA, strlen(EXTNAME_DATA)+1)) {
    /* if this is an IMAGE_FOV, then a STAT extension might be there */
    extension = cpl_fits_find_extension(aFilename, EXTNAME_STAT);
  } else {
    /* otherwise we might have an extension <filtername>_STAT */
    char *statname = cpl_sprintf("%s_STAT", extname);
    extension = cpl_fits_find_extension(aFilename, statname);
    cpl_free(statname);
  }
  if (extension > 0) {
    cpl_errorstate status = cpl_errorstate_get();
    image->stat = cpl_image_load(aFilename, CPL_TYPE_FLOAT, 0, extension);
    if (!cpl_errorstate_is_equal(status)) {
      if (cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) {
        cpl_errorstate_set(status);
        cpl_msg_debug(__func__, "Ignoring empty extension %s in \"%s\"",
                      EXTNAME_STAT, aFilename);
      } else {
        cpl_error_set_message(__func__, MUSE_ERROR_READ_STAT, "Could not load "
                            "extension %s from \"%s\"", EXTNAME_STAT, aFilename);
        cpl_free(extname);
        muse_image_delete(image);
        return NULL;
      }
    }
  }

  /* If present load the DQ extension and encode flagged pixels in the *
   * image data (and possibly the statistics as NaNs.                  */
  if (extname && !strncmp(extname, EXTNAME_DATA, strlen(EXTNAME_DATA)+1)) {
    /* if this is an IMAGE_FOV, then a DQ extension might be there */
    extension = cpl_fits_find_extension(aFilename, EXTNAME_DQ);
  } else {
    /* otherwise we might have an extension <filtername>_DQ */
    char *dqname = cpl_sprintf("%s_DQ", extname);
    extension = cpl_fits_find_extension(aFilename, dqname);
    cpl_free(dqname);
  }
  if (extension > 0) {
    cpl_errorstate status = cpl_errorstate_get();
    image->dq = cpl_image_load(aFilename, CPL_TYPE_INT, 0, extension);
    if (!cpl_errorstate_is_equal(status)) {
      cpl_errorstate_set(status);
      cpl_error_set_message(__func__, MUSE_ERROR_READ_DQ, "Could not load "
                            "extension %s from \"%s\"", EXTNAME_DQ, aFilename);
      muse_image_delete(image);
      cpl_free(extname);
      return NULL;
    }
    muse_image_dq_to_nan(image);
  }

  cpl_free(extname);
  return image;
}
/**@}*/
