/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2014-2018 Laure Piqueras, Simon Conseil, Johan Richard
 *                         (CRAL - Observatoire de Lyon)
 *           (C) 2017-2018 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#include <cpl.h>
#include <math.h>
#include <string.h>

#include "muse_autocalib.h"
#include "muse_instrument.h"

#include "muse_data_format_z.h"
#include "muse_pfits.h"
#include "muse_utils.h"

/*----------------------------------------------------------------------------*
 *                             Macros                                         *
 *----------------------------------------------------------------------------*/
#define MAPIDX(i, s, q) (kMuseNumIFUs * kMuseSlicesPerCCD * ((q)-1)            \
                         + kMuseSlicesPerCCD * ((i)-1) + (s) - 1)

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_autocalib    Autocalibration functions
 *
 * This module contains functions that are used for self calibration of the
 * MUSE instrument.
 */
/*----------------------------------------------------------------------------*/
/**@{*/

/* Compute the arithmetic mean */
static void
muse_autocalib_mean(float *data, int n, double x[3], int *indx)
{
  double mean = 0.0,
         sum_deviation = 0.0;
  int i;
  for (i = 0; i < n; i++) {
    mean += data[indx[i]];
  }
  mean = mean / n;
  for (i = 0; i < n; i++) {
    sum_deviation += (data[indx[i]] - mean) * (data[indx[i]] - mean);
  }
  x[0] = mean;
  x[1] = sqrt(sum_deviation / n);
}

/* Compute median */
static double
muse_autocalib_median(float *data, int n, int *indx)
{
  cpl_array *arr = cpl_array_new(n, CPL_TYPE_FLOAT);
  cpl_size i;
  for (i = 0; i < n; i++) {
    cpl_array_set(arr, i, data[indx[i]]);
  }

  double med = cpl_array_get_median(arr);
  cpl_array_delete(arr);

  return med;
}

/* Compute the arithmetic mean and MAD sigma */
static void
muse_autocalib_mean_mad(float *data, int n, double x[3], int *indx)
{
  double mean = 0.0,
         median = 0.0;
  int i;
  float *work = (float *)cpl_malloc(n * sizeof(float));
  int *ind = (int *)cpl_malloc(n * sizeof(int));

  for (i = 0; i < n; i++) {
    mean += data[indx[i]];
  }
  mean = mean / n;

  median = muse_autocalib_median(data, n, indx);
  for (i = 0; i < n; i++) {
    ind[i] = i;
    work[i] = fabs(data[indx[i]] - median);
  }
  x[0] = mean;
  x[1] = muse_autocalib_median(work, n, ind) * 1.4826;
  cpl_free(ind);
  cpl_free(work);
}

/* Iterative sigma-clipping of array elements *
 * return x[0] = mean, x[1] = std, x[2] = n   *
 * index must be initialized.                 */
static void
muse_autocalib_mean_sigma_clip(float *data, int n, double x[3], int nmax,
                               double nclip_low, double nclip_up, int nstop,
                               int *indx)
{
  double clip_lo, clip_up;
  muse_autocalib_mean(data, n, x, indx);
  x[2] = n;
  double med;
  med =  muse_autocalib_median(data, n, indx);
  clip_lo = med - (nclip_low * x[1]);
  clip_up = med + (nclip_up * x[1]);

  int i, ni = 0;
  for (i = 0; i < n; i++) {
    if ((data[indx[i]] < clip_up) && (data[indx[i]] > clip_lo)) {
      ni = ni + 1;
    }
  }
  if (ni < nstop || ni == n) {
    return;
  }
  if (nmax > 0) {
    ni = 0;
    for (i = 0; i < n; i++) {
      if ((data[indx[i]] < clip_up) && (data[indx[i]] > clip_lo)) {
        indx[ni] = indx[i];
        ni = ni + 1;
      }
    }
    nmax = nmax - 1;
    muse_autocalib_mean_sigma_clip(data, ni, x, nmax, nclip_low, nclip_up,
                                   nstop, indx);
  }
}

/* Iterative MAD sigma-clipping of array elements *
 * return x[0] = median, x[1] = MAD std, x[2] = n */
static void
muse_autocalib_mean_madsigma_clip(float *data, int n, double x[3], int nmax,
                                  double nclip_low, double nclip_up, int nstop,
                                  int *indx)
{
  double clip_lo, clip_up;
  muse_autocalib_mean_mad(data, n, x, indx);
  x[2] = n;
  double med;
  med = muse_autocalib_median(data, n, indx);
  clip_lo = med - (nclip_low * x[1]);
  clip_up = med + (nclip_up * x[1]);

  int i, ni = 0;
  for (i = 0; i < n; i++) {
    if ((data[indx[i]] < clip_up) && (data[indx[i]] > clip_lo)) {
      ni = ni + 1;
    }
  }
  if (ni < nstop || ni == n) {
    return;
  }
  if (nmax > 0) {
    ni = 0;
    for (i = 0; i < n; i++) {
      if ((data[indx[i]] < clip_up) && (data[indx[i]] > clip_lo)) {
        indx[ni] = indx[i];
        ni = ni + 1;
      }
    }
    nmax = nmax - 1;
    muse_autocalib_mean_madsigma_clip(data, ni, x, nmax, nclip_low, nclip_up,
                                      nstop, indx);
  }
}

/* Compute the min and max of data */
static void
muse_autocalib_minmax(float data[], int n, int *indx, float res[])
{
  int i;
  double min, max;

  if (n == 1) {
    res[0] = data[indx[0]];
    res[1] = data[indx[0]];
    return;
  }

  if (data[indx[0]] > data[indx[1]]) {
    max = data[indx[0]];
    min = data[indx[1]];
  } else {
    max = data[indx[1]];
    min = data[indx[0]];
  }

  for (i = 2; i < n; i++) {
    if (data[indx[i]] > max) {
      max = data[indx[i]];
    } else if (data[indx[i]] < min) {
      min = data[indx[i]];
    }
  }
  res[0] = min;
  res[1] = max;
}

/* Compute the min and max of int data */
static void
muse_autocalib_minmax_int(unsigned short data[], int n, int *indx,
                          unsigned int res[])
{
  int i;
  unsigned int min, max;

  if (n == 1) {
    res[0] = data[indx[0]];
    res[1] = data[indx[0]];
    return;
  }

  if (data[indx[0]] > data[indx[1]]) {
    max = data[indx[0]];
    min = data[indx[1]];
  } else {
    max = data[indx[1]];
    min = data[indx[0]];
  }

  for (i = 2; i < n; i++) {
    if (data[indx[i]] > max) {
      max = data[indx[i]];
    } else if (data[indx[i]] < min) {
      min = data[indx[i]];
    }
  }
  res[0] = min;
  res[1] = max;
}

/* Compute the mean flux for a slice.                                 *
 *                                                                    *
 * First, compute the mean flux for each spaxel that has enough       *
 * values (50) along the wavelength axis.                             *
 * Then, compute the mad-clipped mean/std of these fluxes, if we have *
 * at least meanpix spaxels with a valid flux.                        */
static void
muse_autocalib_slice_mean(float *data, unsigned short *xpix, int n, double x[3],
                          int *indx, unsigned int min_pix)
{
  /* Find the min/max xpix values as we don't need *
   * to iterate on spaxels without values.         */
  unsigned int minmax[2];
  muse_autocalib_minmax_int(xpix, n, indx, minmax);

  int nx = (minmax[1] - minmax[0] + 1);
  int *ind = (int *)cpl_malloc(n * sizeof(int));
  float *meanarr = (float *)cpl_malloc(nx * sizeof(float));

  int meancount = 0;
  unsigned int i;
  for (i = minmax[0]; i <= minmax[1]; i++) {
    int j, count = 0;
    for (j = 0; j < n; j++) {
      if (xpix[indx[j]] == i) {
        ind[count++] = indx[j];
      }
    }
    /* if we have at least 50 values for this spaxel, compute its mean flux */
    if (count > 50) {
      muse_autocalib_mean(data, count, x, ind);
      meanarr[meancount] = x[0];
      meancount++;
    }
  }
  if (meancount < (int)min_pix) {
    x[0] = 0;
    x[1] = 0;
    x[2] = 0;
  } else {
    int j;
    for (j = 0; j < meancount; j++) {
      ind[j] = j;
    }
    muse_autocalib_mean_madsigma_clip(meanarr, meancount, x, 15, 3, 3, 15, ind);
  }

  cpl_free(ind);
  cpl_free(meanarr);
}

/* apply the correction factor to the data buffers of a pixel table */
static void
muse_autocalib_applycorr(float *data, float *stat, int npix, unsigned char *ifu,
                         unsigned char *sli, unsigned int *quad, float *corr,
                         const char *idstring)
{
  cpl_msg_debug(idstring, "Applying corrections...");
  cpl_size n;
  #pragma omp parallel for default(none)                                       \
          shared(corr, data, ifu, npix, quad, sli, stat, idstring)
  for (n = 0; n < npix; n++) {
    cpl_size k = MAPIDX(ifu[n], sli[n], quad[n]);
    float c = corr[k];
    data[n] *= c;
    stat[n] *= c * c;
  }
}

/*---------------------------------------------------------------------------*/
/**
  @brief   Correct the background level of the slices (port from MPDAF).
  @param   aPixtable   input pixel table
  @param   aClip       clipping threshold for slice corrections in one IFU
  @param   aFactors    pointer to output table containing the correction factors
                       (optional, can be NULL)
  @return  CPL_ERROR_NONE on success or a cpl_error_code on failure

  This requires a pixel table that is not sky-subtracted. It uses the mean sky
  level as a reference, and compute a multiplicative correction to apply to each
  slice to bring its background level to the reference one.

  Sources are expected to be masked in that only the sky background pixels
  (table rows) in aPixtable are selected that this algorithms is supposed to
  work on.

  @remark  This function adds a FITS header (@ref MUSE_HDR_PT_AUTOCAL_NAME) with
           the string "slice-median" to the pixel table, so that other functions
           can detect, if this type of autocalibration has already been carried
           out. It will return without processing or error, if this keyword
           already exists in the input pixel table, and the string contains a
           known method.
           The comment of that header starts with the value of aClip.

  @error{set and return CPL_ERROR_NULL_INPUT, aPixtable is NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_autocalib_slice_median(muse_pixtable *aPixtable, double aClip,
                            muse_table **aFactors)
{
  cpl_ensure_code(aPixtable, CPL_ERROR_NULL_INPUT);
  /* check, if an autocalibration was already applied to this pixel table */
  const char *kwstring = NULL;
  if (cpl_propertylist_has(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME)) {
    kwstring = cpl_propertylist_get_string(aPixtable->header,
                                           MUSE_HDR_PT_AUTOCAL_NAME);
  }
  if (kwstring && (!strncmp(kwstring, "slice-median", 13) ||
                   !strncmp(kwstring, "user", 5))) {
    cpl_msg_info(__func__, "pixel table already auto-calibrated (method %s): "
                 "skipping correction", kwstring);
    return CPL_ERROR_NONE;
  }

  /* --- Hardcoded Parameters -----------------------------------------------*/
  /* XXX use real parameters for these? If so, we    *
   *     need to make the isAO branch below smarter! */
  float lbdabins[] = { 0, 5000, 5265, 5466, 5658, 5850, 6120, 6440, 6678,
                       6931, 7211, 7450, 7668, 7900, 8120, 8330, 8565, 8731,
                       9012, 9275, 10000 };
  unsigned short nlbin = 20;
  /* Minimum number of pixels for which we have a flux in one slice, *
   * and maximum number that can happen at all in the MUSE slice.    */
  unsigned int min_pix_per_slice = 20,
               max_pix_per_slice = (kMuseOutputYTop
                                    * ceil(kMuseSliceHiLikelyWidth));

  /* Clipping parameters */
  int nmax = 15, nstop = 20;
  double nclip_low = 3.0, nclip_up = 3.0;
  /*-------------------------------------------------------------------------*/

  cpl_boolean isAO = muse_pfits_get_mode(aPixtable->header)
                     >= MUSE_MODE_WFM_AO_E;
  if (isAO) {
    /* only one range around the NaD gap, remove the 5850 entry */
    memmove(lbdabins+5, lbdabins+6, 15 * sizeof(float));
    nlbin--;
  }

  cpl_msg_info(__func__, "Running self-calibration, using %d lambda ranges "
               "(%s mode)", nlbin, isAO ? "AO" : "non-AO");
  double cputime = cpl_test_get_cputime(),
         walltime = cpl_test_get_walltime();

  double x[3], x1[3], x2[3], *refx;
  float tot_flux, refflux, minmax[2], oldcorr, meanq, threshq;

  unsigned char *ifu, *sli;
  unsigned short *xpix;
  muse_pixtable_origin_decode_all(aPixtable, &xpix, NULL, &ifu, &sli);
  cpl_size npix = muse_pixtable_get_nrow(aPixtable);

  /* quad is the wavelength bin index. This is a poor name inherited *
   * from a previous version of the autocalib code...                */
  unsigned int *quad = (unsigned int *)cpl_malloc(npix * sizeof(unsigned int));

  /* For each slice and wavelength bin, store the number of *
   * values, the correction, and the indices of the values  */
  int *npts = (int *)cpl_malloc(kMuseNumIFUs * kMuseSlicesPerCCD * nlbin
                                * sizeof(int));
  float *corr = (float *)cpl_malloc(kMuseNumIFUs * kMuseSlicesPerCCD * nlbin
                                    * sizeof(float));
  int *indmap[nlbin * kMuseNumIFUs * kMuseSlicesPerCCD];

  cpl_size k;
  for (k = 0; k < kMuseNumIFUs * kMuseSlicesPerCCD * nlbin; k++) {
    npts[k] = 0;
    corr[k] = 1.0;
    indmap[k] = (int *)cpl_malloc(max_pix_per_slice * sizeof(int));
  }

  float *data = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_DATA),
        *stat = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_STAT),
        *lbda = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_LAMBDA);

  cpl_boolean debug = getenv("MUSE_DEBUG_AUTOCALIB") ? CPL_TRUE : CPL_FALSE;

  /* loop through all pixels and fill the remaining buffers */
  cpl_msg_debug(__func__, "Computing lambda indices...");
  cpl_size n;
#if _OPENMP >= 201107 /* needed for atomic capture, see below */
  #pragma omp parallel for default(none) private(k)                            \
          shared(aPixtable, data, ifu, indmap, lbda, lbdabins, nlbin, npix,    \
                 npts, quad, sli)
#endif
  for (n = 0; n < npix; n++) {
    unsigned short q;
    for (q = 1; q < nlbin+1; q++) {
      if ((lbda[n] >= lbdabins[q-1]) && (lbda[n] < lbdabins[q])) {
        quad[n] = q;
        break;
      }
    }
    if (cpl_table_is_selected(aPixtable->table, n)) {
      k = MAPIDX(ifu[n], sli[n], quad[n]);
      int i;
      /* atomic capture is new in OpenMP v3.1 (or _OPENMP=201107), */
#if _OPENMP >= 201107 /* needed for atomic capture */
      #pragma omp atomic capture
#endif
      i = npts[k]++;
      indmap[k][i] = n;
    }
  }

  float *slice_flux = (float *)cpl_malloc(kMuseSlicesPerCCD * kMuseNumIFUs
                                          * sizeof(float)),
         *ifu_flux = (float *)cpl_malloc(kMuseNumIFUs * 2 * sizeof(float));
  int *slice_ind1 = (int *)cpl_malloc(kMuseSlicesPerCCD * sizeof(int)),
      *slice_ind2 = (int *)cpl_malloc(kMuseSlicesPerCCD * sizeof(int)),
      *tot_ind = (int *)cpl_malloc(kMuseNumIFUs * kMuseSlicesPerCCD
                                   * sizeof(int));
  cpl_msg_debug(__func__, "Computing reference levels and corrections...");
  unsigned short q;
  for (q = 0; q < nlbin; q++) {
    if (debug) {
      cpl_msg_debug(__func__, "Computing reference levels (lambda range: "
                    "%.1f - %.1f Angstrom)...", lbdabins[q], lbdabins[q+1]);
    }
    int i, tot_count = 0;
    for (i = 0; i < kMuseNumIFUs; i++) {
      if (debug) {
        cpl_msg_debug(__func__, "- IFU %02d", i+1);
      }
      int slice_count1 = 0,
          slice_count2 = 0;
      unsigned short s;
      for (s = 0; s < kMuseSlicesPerCCD; s++) {
        k = MAPIDX(i+1, s+1, q+1);
        int slidx = kMuseSlicesPerCCD * i + s;
        if (npts[k] > 100) {
          /* Compute the mean flux of the ifu/slice/wavelength bin,  *
           * if we have at least 100 values, and if we have at least *
           * min_pix_per_slice spaxels with a flux.                  */
          muse_autocalib_slice_mean(data, xpix, npts[k], x, indmap[k],
                                    min_pix_per_slice);
          if (x[2] > min_pix_per_slice) {
            slice_flux[slidx] = x[0];
            if (s < kMuseSlicesPerCCD/2) {
              slice_ind1[slice_count1++] = slidx;
            } else {
              slice_ind2[slice_count2++] = slidx;
            }
            tot_ind[tot_count++] = slidx;
          } else {
            slice_flux[slidx] = NAN;
          }
        } else {
          slice_flux[slidx] = NAN;
        }
        if (debug) {
          cpl_msg_debug(__func__, "  - SLICE %02hu : %f (%d)",
                        s+1, slice_flux[slidx], npts[k]);
        }
      } /* for s */

      /* Compute the mean flux for each half-IFU, as the mean of the *
       * slice fluxes. */
      if (!slice_count1) {
        ifu_flux[2 * i] = 0.0;
      } else {
        muse_autocalib_mean_madsigma_clip(slice_flux, slice_count1, x, nmax,
                                          nclip_low, nclip_up, nstop,
                                          slice_ind1);
        ifu_flux[2 * i] = x[0];

        if (debug) {
          muse_autocalib_minmax(slice_flux, slice_count1, slice_ind1, minmax);
          cpl_msg_debug(__func__, "  - 1: Min max = %f %f / Mean = %f (%f, %d)",
                        minmax[0], minmax[1], x[0], x[1], (int)x[2]);
        }
        if (isnan(x[0])) {
          cpl_msg_warning(__func__, "Mean IFU flux is NAN (IFU %02d, lambda range"
                          " %.1f - %.1f Angstrom)", i+1, lbdabins[q],
                          lbdabins[q+1]);
        }
      }

      /* Same for the second half-IFU. */
      if (!slice_count2) {
        ifu_flux[2 * i + 1] = 0.0;
      } else {
        muse_autocalib_mean_madsigma_clip(slice_flux, slice_count2, x, nmax,
                                          nclip_low, nclip_up, nstop,
                                          slice_ind2);
        ifu_flux[2 * i + 1] = x[0];

        if (debug) {
          muse_autocalib_minmax(slice_flux, slice_count2, slice_ind2, minmax);
          cpl_msg_debug(__func__, "  - 2: Min max = %f %f / Mean = %f (%f, %d)",
                        minmax[0], minmax[1], x[0], x[1], (int)x[2]);
        }
        if (isnan(x[0])) {
          cpl_msg_warning(__func__, "Mean IFU flux is NAN (IFU %02d, lambda range"
                          " %.1f - %.1f Angstrom)", i+1, lbdabins[q],
                          lbdabins[q+1]);
        }
      }

      /* if low number of valid slices test if the other half-IFU has more *
       * valid slice fluxes and use them instead                           */
      if ((slice_count1 < kMuseSlicesPerCCD/3) &&
          (slice_count2 > kMuseSlicesPerCCD/3)) {
        cpl_msg_debug(__func__, "Use right-half flux value of IFU %d for "
                      "left-half", i+1);
        ifu_flux[2 * i]=ifu_flux[2 * i + 1];
      }

      if ((slice_count1 > kMuseSlicesPerCCD/3) &&
          (slice_count2 < kMuseSlicesPerCCD/3)) {
        cpl_msg_debug(__func__, "Use left-half flux value of IFU %d for "
                      "right-half", i+1);
        ifu_flux[2 * i + 1]=ifu_flux[2 * i];
      }

      /* Use the mean ifu flux for slices with invalid value. */
      for (s = 0; s < kMuseSlicesPerCCD; s++) {
        int slidx = kMuseSlicesPerCCD * i + s;
        if (isnan(slice_flux[slidx])) {
          if (s < kMuseSlicesPerCCD/2) {
            slice_flux[slidx] = ifu_flux[2 * i];
          } else {
            slice_flux[slidx] = ifu_flux[2 * i + 1];
          }
        }
      } /* for s */
    } /* for i */
    if (!tot_count) {
      cpl_msg_warning(__func__, "No values in this lambda range (%.1f - %.1f "
                      "Angstrom)", lbdabins[q], lbdabins[q+1]);
      continue;
    }
    if (debug) {
      muse_autocalib_minmax(slice_flux, tot_count, tot_ind, minmax);
      cpl_msg_debug(__func__, "- Min max : %f %f", minmax[0], minmax[1]);
    }

    /* Compute the total flux in the bin, as the sigma-clipped mean *
     * of the slice fluxes.                                         */
    muse_autocalib_mean_sigma_clip(slice_flux, tot_count, x, nmax, nclip_low,
                                   nclip_up, nstop, tot_ind);
    if (debug) {
      cpl_msg_debug(__func__, "- Total flux : %f (%f, %d)", x[0], x[1],
                    (int)x[2]);
    }
    tot_flux = x[0];

    /* Compute the correction factors.                                     *
     * First, compute the ratio between the total flux and the slice flux. *
     * Then, apply a mad-sigma clipping to these corrections, to remove    *
     * strongest ones.                                                     */
    if (debug) {
      cpl_msg_debug(__func__, "Computing corrections...");
    }
    for (i = 0; i < kMuseNumIFUs; i++) {
      if (debug) {
        cpl_msg_debug(__func__, "- IFU %02d", i+1);
      }
      int slice_count1 = 0,
          slice_count2 = 0;
      unsigned short s;
      for (s = 0; s < kMuseSlicesPerCCD; s++) {
        k = MAPIDX(i+1, s+1, q+1);
        int slidx = kMuseSlicesPerCCD * i + s;
        if (npts[k] != 0) {
          if (s < kMuseSlicesPerCCD/2) {
            slice_ind1[slice_count1++] = k;
          } else {
            slice_ind2[slice_count2++] = k;
          }
        }
        if (slice_flux[slidx] > 0) {
          corr[k] = tot_flux / slice_flux[slidx];
        }
        if (debug) {
          cpl_msg_debug(__func__, "  - SLICE %02hu : %f", s+1, corr[k]);
        }
      } /* for s */
      if ((slice_count1 + slice_count2) == 0) {
        cpl_msg_warning(__func__, "No values in this IFU (%02d, lambda range: "
                        "%.1f - %.1f Angstrom)", i+1, lbdabins[q],
                        lbdabins[q+1]);
        continue;
      }

      if (slice_count1) {
        muse_autocalib_mean_madsigma_clip(corr, slice_count1, x1, nmax,
                                          nclip_low, nclip_up, nstop,
                                          slice_ind1);
        if (debug) {
          muse_autocalib_minmax(corr, slice_count1, slice_ind1, minmax);
          cpl_msg_debug(__func__, "  - 1: Min max = %f %f / Mean = %f (%f, %d)",
                        minmax[0], minmax[1], x1[0], x1[1], (int)x1[2]);
        }
      }

      if (slice_count2) {
        muse_autocalib_mean_madsigma_clip(corr, slice_count2, x2, nmax,
                                          nclip_low, nclip_up, nstop,
                                          slice_ind2);
        if (debug) {
          muse_autocalib_minmax(corr, slice_count2, slice_ind2, minmax);
          cpl_msg_debug(__func__, "  - 2: Min max = %f %f / Mean = %f (%f, %d)",
                        minmax[0], minmax[1], x2[0], x2[1], (int)x2[2]);
        }
      }

      if (debug) {
        cpl_msg_debug(__func__, "  - Checking slice corrections "
                      "(%.1f sigma clip)...", aClip);
      }
      for (s = 0; s < kMuseSlicesPerCCD; s++) {
        k = MAPIDX(i+1, s+1, q+1);
        if (s < kMuseSlicesPerCCD/2) {
          if (!slice_count1) {
            continue;
          }
          refx = x1;
          refflux = ifu_flux[2 * i];
        } else {
          if (!slice_count2) {
            continue;
          }
          refx = x2;
          refflux = ifu_flux[2 * i + 1];
        }
        if (fabs(corr[k] - refx[0]) > aClip * refx[1]) {
          oldcorr = corr[k];
          corr[k] = tot_flux / refflux;
          if (debug) {
            cpl_msg_debug(__func__, "  - SLICE %02hu : %f -> Using IFU Mean : "
                          "%f", s+1, oldcorr, corr[k]);
          }
        }
      } /* for s */
    } /* for i */
  } /* for q */

  /* create optional output table with determined properties */
  if (aFactors) {
    *aFactors = muse_table_new();
    (*aFactors)->table = muse_cpltable_new(muse_autocal_results_def,
                                         kMuseNumIFUs * kMuseSlicesPerCCD * nlbin);
    /* create header based on the input pixel table */
    (*aFactors)->header = cpl_propertylist_duplicate(aPixtable->header);
    cpl_propertylist_erase_regexp((*aFactors)->header, MUSE_HDR_PT_REGEXP, 0);
    /* save original correction factor as extra column */
    cpl_table_new_column((*aFactors)->table, "corr_orig", CPL_TYPE_DOUBLE);
  } /* if aFactors */

  cpl_msg_debug(__func__, "Cleaning and recording corrections...");
  int nnan = 0;
  for (k = 0; k < kMuseNumIFUs * kMuseSlicesPerCCD * nlbin; k++) {
    if (isnan(corr[k])) {
      corr[k] = 1.;
      nnan++;
    }
  }
  if (nnan > 0) {
    cpl_msg_warning(__func__, "%d of %d wavelength bins in %d slices stay "
                    "uncorrected!", nnan, kMuseNumIFUs * kMuseSlicesPerCCD * nlbin,
                    kMuseNumIFUs * kMuseSlicesPerCCD);
  }
  int i;
  for (i = 1; i <= kMuseNumIFUs; i++) {
    unsigned short s;
    for (s = 1; s <= kMuseSlicesPerCCD; s++) {
      for (q = 1; q <= nlbin; q++) {
        k = MAPIDX(i, s, q);
        if (aFactors) {
          /* record slice properties in the table */
          cpl_table_set_int((*aFactors)->table, "ifu", k, i);
          cpl_table_set_int((*aFactors)->table, "sli", k, s);
          cpl_table_set_int((*aFactors)->table, "quad", k, q);
          cpl_table_set_int((*aFactors)->table, "npts", k, npts[k]);
          cpl_table_set_double((*aFactors)->table, "corr", k, corr[k]);
          /* store wavelength range in header */
          if (i == 1 && s == 1) {
            char *kw1 = cpl_sprintf("ESO DRS MUSE LAMBDA%d MIN", q),
                 *kw2 = cpl_sprintf("ESO DRS MUSE LAMBDA%d MAX", q);
            cpl_propertylist_append_float((*aFactors)->header, kw1, lbdabins[q-1]);
            cpl_propertylist_append_float((*aFactors)->header, kw2, lbdabins[q]);
            cpl_propertylist_set_comment((*aFactors)->header, kw1,
                                         "[Angstrom] lower end of wavelength range");
            cpl_propertylist_set_comment((*aFactors)->header, kw2,
                                         "[Angstrom] upper end of wavelength range");
            cpl_free(kw1);
            cpl_free(kw2);
          } /* if first slice */
        }

        cpl_size kprev, knext;
        if (q == 1) {
          kprev = MAPIDX(i, s, q+1);
          knext = MAPIDX(i, s, q+2);
        } else if (q == nlbin) {
          kprev = MAPIDX(i, s, q-1);
          knext = MAPIDX(i, s, q-2);
        } else {
          kprev = MAPIDX(i, s, q-1);
          knext = MAPIDX(i, s, q+1);
        }
        meanq = (corr[kprev] + corr[knext]) / 2.;
        threshq = CPL_MAX(0.03, 3 * fabsf(corr[kprev] - corr[knext]));
        if (fabsf(corr[k] - meanq) > threshq) {
          if (aFactors) { /* store both original and revised factor */
            cpl_table_set_double((*aFactors)->table, "corr_orig", k, corr[k]);
            cpl_table_set_double((*aFactors)->table, "corr", k, meanq);
          }
          if (debug) {
            cpl_msg_debug(__func__, "- %02d / %02hu / %02hu : %f -> %f",
                          i, s, q, corr[k], meanq);
          }
          corr[k] = meanq;
        } /* if */
      } /* for q */
    } /* for s */
  } /* for i */

  muse_autocalib_applycorr(data, stat, npix, ifu, sli, quad, corr, __func__);
  muse_utils_memory_dump("ac_max");

  /* cleanup the remaining allocations */
  for (k = 0; k < kMuseNumIFUs * kMuseSlicesPerCCD * nlbin; k++) {
    cpl_free(indmap[k]);
  }
  cpl_free(slice_flux);
  cpl_free(ifu_flux);
  cpl_free(slice_ind1);
  cpl_free(slice_ind2);
  cpl_free(tot_ind);
  cpl_free(ifu);
  cpl_free(sli);
  cpl_free(quad);
  cpl_free(xpix);
  cpl_free(npts);
  cpl_free(corr);

  /* add the header */
  cpl_propertylist_update_string(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME,
                                 "slice-median");
  char *comment = cpl_sprintf(MUSE_HDR_PT_AUTOCAL_COMMENT, aClip);
  cpl_propertylist_set_comment(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME,
                               comment);
  cpl_free(comment);

  cpl_msg_debug(__func__, "... done. Self-calibration took %gs (CPU time), %gs "
                "(wall-clock)", cpl_test_get_cputime() - cputime,
                cpl_test_get_walltime() - walltime);

  return CPL_ERROR_NONE;
} /* muse_autocalib_slice_median() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Apply user self-calibration corrections to a pixel table.
  @param   aPixtable   input pixel table
  @param   aFactors    input table containing the correction factors
  @return  CPL_ERROR_NONE on success or a cpl_error_code on failure

  This uses the user-provided multiplicative corrections to apply to each slice.

  The format of aFactors is expected to be the same as the one produced by
  @ref muse_autocalib_slice_median().

  @remark  This function adds a FITS header (@ref MUSE_HDR_PT_AUTOCAL_NAME) with
           the string "user" to the pixel table, so that other functions
           can detect, if this manual autocalibration has already been carried
           out. It will return without processing or error, if this keyword
           already exists in the input pixel table, and the string contains a
           known method.

  @error{set and return CPL_ERROR_NULL_INPUT,
         aPixtable or aFactors or one of their components are NULL}
  @error{set and return CPL_ERROR_DATA_NOT_FOUND,
         aFactors does not contain a column "corr" in its table component}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_autocalib_apply(muse_pixtable *aPixtable, muse_table *aFactors)
{
  cpl_ensure_code(aPixtable && aPixtable->table &&
                  aFactors && aFactors->header && aFactors->table,
                  CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(cpl_table_has_column(aFactors->table, "corr"),
                  CPL_ERROR_DATA_NOT_FOUND);
  /* check, if an autocalibration was already applied to this pixel table */
  const char *kwstring = NULL;
  if (cpl_propertylist_has(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME)) {
    kwstring = cpl_propertylist_get_string(aPixtable->header,
                                           MUSE_HDR_PT_AUTOCAL_NAME);
  }
  if (kwstring && (!strncmp(kwstring, "slice-median", 13) ||
                   !strncmp(kwstring, "user", 5))) {
    cpl_msg_info(__func__, "pixel table already auto-calibrated (method %s): "
                 "skipping correction", kwstring);
    return CPL_ERROR_NONE;
  }

  /* read lambda_bins from aFactors header */
  int nlbin = 1;
  /* first search for number of lambda bins */
  while (nlbin < 1000) { /* arbitrary limit, should be 19 or 20... */
    char *kw1 = cpl_sprintf("ESO DRS MUSE LAMBDA%d MIN", nlbin);
    if (!cpl_propertylist_has(aFactors->header, kw1)) {
      cpl_free(kw1);
      break;
    }
    nlbin++;
    cpl_free(kw1);
  }
  nlbin--;
  if (!nlbin) {
    char *str = cpl_sprintf("No keywords \"ESO DRS MUSE LAMBDAi MIN/MAX\" found"
                            " in %s header!", MUSE_TAG_ACAL_FACTORS);
    cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_INPUT, "%s", str);
    cpl_msg_error(__func__, "%s", str);
    cpl_free(str);
    return CPL_ERROR_ILLEGAL_INPUT;
  }
  cpl_boolean isAO = muse_pfits_get_mode(aPixtable->header)
                   >= MUSE_MODE_WFM_AO_E;
  cpl_msg_info(__func__, "Running self-calibration, using %d lambda ranges "
               "(%s data) and user table", nlbin, isAO ? "AO" : "non-AO");
  int nrow = cpl_table_get_nrow(aFactors->table),
      nexpected = kMuseSlicesPerCCD * kMuseNumIFUs * nlbin;
  if (nrow != nexpected) {
    cpl_msg_warning(__func__, "%s contains %d instead of %d rows!",
                    MUSE_TAG_ACAL_FACTORS, nrow, nexpected);
  }

  /* now set up the buffer and fill it with the wavelength ranges */
  float *lbdabins = (float *)cpl_malloc((nlbin+1) * sizeof(float));
  unsigned short q = 0;
  for (q = 1; q < nlbin+1; q++) {
    char *kw1 = cpl_sprintf("ESO DRS MUSE LAMBDA%d MIN", q),
         *kw2 = cpl_sprintf("ESO DRS MUSE LAMBDA%d MAX", q);
    lbdabins[q-1] = cpl_propertylist_get_double(aFactors->header, kw1);
    lbdabins[q] = cpl_propertylist_get_double(aFactors->header, kw2);
    cpl_free(kw1);
    cpl_free(kw2);
  }

  unsigned char *ifu, *sli;
  unsigned short *xpix;
  muse_pixtable_origin_decode_all(aPixtable, &xpix, NULL, &ifu, &sli);

  float *data = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_DATA),
        *stat = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_STAT),
        *lbda = cpl_table_get_data_float(aPixtable->table, MUSE_PIXTABLE_LAMBDA);

  cpl_size n, npix = muse_pixtable_get_nrow(aPixtable);
  unsigned int *quad = (unsigned int *)cpl_malloc(npix * sizeof(unsigned int));

  /* loop through all pixels and fill the remaining buffers */
  cpl_msg_debug(__func__, "Computing lambda indices...");
  #pragma omp parallel for default(none) private(q)                            \
          shared(lbda, lbdabins, nlbin, npix, quad)
  for (n = 0; n < npix; n++) {
    for (q = 1; q < nlbin+1; q++) {
      if ((lbda[n] >= lbdabins[q-1]) && (lbda[n] < lbdabins[q])) {
        quad[n] = q;
        break;
      }
    }
  }
  cpl_free(lbdabins);

  cpl_table_cast_column(aFactors->table, "corr", "corr_float", CPL_TYPE_FLOAT);
  float *corr = cpl_table_get_data_float(aFactors->table, "corr_float");
  muse_autocalib_applycorr(data, stat, npix, ifu, sli, quad, corr, __func__);
  cpl_table_erase_column(aFactors->table, "corr_float");
  cpl_free(xpix);
  cpl_free(ifu);
  cpl_free(sli);
  cpl_free(quad);

  /* add the header */
  cpl_propertylist_update_string(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME,
                                 "user");
  cpl_propertylist_set_comment(aPixtable->header, MUSE_HDR_PT_AUTOCAL_NAME,
                               "used user table for slice autocalibration");

  return CPL_ERROR_NONE;
} /* muse_autocalib_apply() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Create Select spaxels to be considered as sky for autocalibration.
  @param   aImage      MUSE (white-light) image of the field of view
  @param   aNSigma     Sigma level for the statistics to mask
  @param   aQCPrefix   prefix for the QC keywords (optional, can be NULL)
  @return  CPL_ERROR_NONE on success or a cpl_error_code on failure

  Use thresholding and binary filtering to create a mask of sky regions.  This
  function evolved from from @ref muse_sky_create_skymask() but creates more
  contiguous regions to be used for the selection of pixels that get included in
  the autocalibration process.

  If aQCPrefix is given, two parameters are added to the output mask header,
  that can be used for QC.

  @error{return CPL_ERROR_NULL_INPUT, aImage is NULL}
 */
/*---------------------------------------------------------------------------*/
muse_mask *
muse_autocalib_create_mask(muse_image *aImage, double aNSigma,
                           const char *aQCPrefix)
{
  cpl_ensure(aImage, CPL_ERROR_NULL_INPUT, NULL);

  muse_image_reject_from_dq(aImage);

  double median, mad;
  median = cpl_image_get_mad(aImage->data, &mad);
  double t0 = median - aNSigma * mad,
         t1 = median + aNSigma * mad;
  cpl_msg_info(__func__, "Computing sky mask (median = %g, mad = %g)", median,
               mad);

  /* create output mask with pixels between min and threshold */
  muse_mask *selected = muse_mask_new();
  selected->mask = cpl_mask_threshold_image_create(aImage->data, t0, t1);
  cpl_mask_not(selected->mask);
#if 0
  muse_mask_save(selected, "muse_autocalib_mask1.fits");
#endif

  /* Use a morphological opening to remove the single pixel detections */
  /* These two should not be able to fail */
  cpl_mask *kernel = cpl_mask_new(3, 3);
  cpl_mask_not(kernel);

  cpl_mask_filter(selected->mask, selected->mask, kernel, CPL_FILTER_OPENING,
                  CPL_BORDER_COPY);
  cpl_mask *tmp = cpl_mask_duplicate(selected->mask);
  cpl_mask_filter(tmp, selected->mask, kernel, CPL_FILTER_DILATION,
                  CPL_BORDER_NOP);
  cpl_mask_filter(selected->mask, tmp, kernel, CPL_FILTER_DILATION,
                  CPL_BORDER_NOP);
  cpl_mask_delete(tmp);
  cpl_mask_delete(kernel);

  cpl_mask_not(selected->mask);

  /* add threshold as QC parameter */
  selected->header = cpl_propertylist_duplicate(aImage->header);
  if (aQCPrefix) {
    char keyword[KEYWORD_LENGTH];
    snprintf(keyword, KEYWORD_LENGTH, "%s LOWLIMIT", aQCPrefix);
    cpl_propertylist_append_double(selected->header, keyword, t0);
    snprintf(keyword, KEYWORD_LENGTH, "%s THRESHOLD", aQCPrefix);
    cpl_propertylist_append_double(selected->header, keyword, t1);
  }

#if 0
  muse_mask_save(selected, "muse_autocalib_mask2.fits");
#endif

  return selected;
} /* muse_autocalib_create_mask() */

/**@}*/
