/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2018 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#define STORE_SLIT_WIDTH
#define STORE_BIN_WIDTH

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#include <math.h>
#include <string.h>
#include <fenv.h>
#include <cpl.h>

#include "muse_lsf.h"

#include "muse_instrument.h"
#include "muse_optimize.h"
#include "muse_pfits.h"
#include "muse_pixtable.h"
#include "muse_quality.h"
#include "muse_resampling.h"
#include "muse_tracing.h"
#include "muse_utils.h"
#include "muse_wavecalib.h"

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_lsf          Line Spread Function related functions
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/*----------------------------------------------------------------------------*/
/**
   @private
   @brief Check the pixels for one arc line.
   @param aPixtable   the pixel table with pixels around the line
   @param aLambda     the arc line reference wavelength

   The basic check is done with a gaussian fit, which also gives the
   total flux for this line. Lines with a low flux, and lines with an
   implausible Gaussian width are completely ignored (the function
   returns 0). Otherwise, pixels that deviate too much from the
   gaussian are deleted from the pixel table.

   Finally, new columns <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt> are added to the pixel table and set with
   the constant values of the aLambda parameter, the calculated total
   line flux, and the calculated continuum background.
 */
/*----------------------------------------------------------------------------*/
static cpl_size
muse_lsf_check_arc_line(cpl_table *aPixtable, double aLambda)
{
  cpl_size nrows = cpl_table_get_nrow(aPixtable);
  /* If we have too few data pointe, remove the line */
  if (nrows <= 100) {
    return 0;
  }
  cpl_errorstate prestate = cpl_errorstate_get();
  /* get origin before any more (or all?!) rows are removed */
  uint32_t origin = cpl_table_get_int(aPixtable, MUSE_PIXTABLE_ORIGIN, 0, NULL);

  /* wrap table columns into vectors for the Gaussian fit */
  cpl_vector *pos = cpl_vector_new(nrows);
  cpl_vector *val = cpl_vector_new(nrows);
  cpl_vector *err = cpl_vector_new(nrows);
  double *d_pos = cpl_vector_get_data(pos);
  double *d_val = cpl_vector_get_data(val);
  double *d_err = cpl_vector_get_data(err);
  float *d_lambda = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_LAMBDA);
  float *d_data = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_DATA);
  float *d_stat = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_STAT);
  cpl_size i, s = 1;
  double bglevel = 0;
  for (i = 0; i < nrows; i++) {
    if (fabs(d_lambda[i] - aLambda) > 5.5) {
      bglevel += d_data[i];
      s += 1;
    }
    d_pos[i] = d_lambda[i];
    d_val[i] = d_data[i];
    d_err[i] = sqrt(d_stat[i]);
  }
  bglevel /= s;
  double meansigma = sqrt(cpl_table_get_column_mean(aPixtable, MUSE_PIXTABLE_STAT));
  double sigma, area, mse;
  double xc = aLambda;
  cpl_fit_mode fit_pars = CPL_FIT_STDEV | CPL_FIT_AREA | CPL_FIT_CENTROID;
  cpl_vector_fit_gaussian(pos, NULL, val, NULL, fit_pars, &xc, &sigma,
                          &area, &bglevel, &mse, NULL, NULL);

  /* Subtract the fit */
  for (i = 0; i < nrows; i++) {
    double e = (d_pos[i] - xc)/sigma;
    d_val[i] -= area / (CPL_MATH_SQRT2PI * sigma) * exp(-e*e/2) - bglevel;
  }

  /* Remove all pixels that are too far from the gaussian estimate */
  double maxerror = 0.5 * sqrt(area) + 0.15 * area;
  cpl_size inv = 0;
  cpl_table_unselect_all(aPixtable);
  for (i = 0; i < nrows; i++) {
    if ((d_val[i] < -maxerror) || (d_val[i] > maxerror)) {
      inv++;
      cpl_table_select_row(aPixtable, i);
    }
  }
  cpl_vector_delete(pos);
  cpl_vector_delete(val);
  cpl_vector_delete(err);
  cpl_table_erase_selected(aPixtable);

  /* If the fit failed, just ignore the errors, and remove the line */
  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_errorstate_set(prestate);
    return 0;
  }

  nrows = cpl_table_get_nrow(aPixtable);
  if (nrows == 0) {
    return 0;
  }

  cpl_table_fill_column_window_float(aPixtable, "line_lambda",
                                     0, nrows, xc);
  cpl_table_fill_column_window_float(aPixtable, "line_flux",
                                     0, nrows, area);
  cpl_table_fill_column_window_float(aPixtable, "line_background",
                                     0, nrows, bglevel);

  /* If the flux (compared to noise) is too low, silently remove the line */
  if (area/meansigma < 50) {
    return 0;
  }
  /* If the estimated line width is too far away from our expectation,
     remove the line */
  if ((sigma > 3) || (sigma < 0.7)) {
    int i_ifu = muse_pixtable_origin_get_ifu(origin);
    int i_slice = muse_pixtable_origin_get_slice(origin);
    cpl_msg_debug(__func__, "Slice %2i.%02i: "
                  "Ignoring line %.1f with implausible width %f (flux=%.0f)",
                  i_ifu, i_slice, aLambda, sigma, area);
    return 0;
  }
  return nrows;
} /* muse_lsf_check_arc_line() */

/*----------------------------------------------------------------------------*/
/**
   @private
   @brief Select the pixels for one arc line
   @param aPixtable The pixel table for one slice
   @param aLambda   The arc line reference wavelength
   @param aWindow     Wavelength window around each line
   @return The pixel table with pixels selected by this arc line, or
           NULL if the line could not be found.

   From the pixel table, the pixels around the specified wavelength
   are selected. Then, a simple gaussian fit is made and the pixels
   that are too far from the gaussian are removed as a simple
   rejection of cosmics. Also, the reference wavelength and the
   estimated flux (from the gaussian fit) are added as additional
   columns <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt>

   If no line is found, NULL is returned.
 */
/*----------------------------------------------------------------------------*/
static cpl_table *
muse_lsf_select_arc_line(muse_pixtable *aPixtable, double aLambda,
                         double aWindow)
{
  cpl_size low = muse_cpltable_find_sorted(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                                           aLambda - aWindow);
  cpl_size high = muse_cpltable_find_sorted(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                                            aLambda + aWindow);
  if (high == low) {
    return NULL;
  }
  cpl_table *sel = cpl_table_extract(aPixtable->table, low + 1, high - low);
  cpl_table_select_all(sel);
  cpl_table_and_selected_int(sel, "dq", CPL_NOT_EQUAL_TO, 0);
  cpl_table_erase_selected(sel);
  if (cpl_table_get_nrow(sel) == 0) {
    cpl_table_delete(sel);
    return NULL;
  }

  cpl_table_new_column(sel, "line_lambda", CPL_TYPE_FLOAT);
  cpl_table_new_column(sel, "line_flux", CPL_TYPE_FLOAT);
  cpl_table_new_column(sel, "line_background", CPL_TYPE_FLOAT);

  cpl_size nrows = muse_lsf_check_arc_line(sel, aLambda);
  if (nrows == 0) {
    cpl_table_delete(sel);
    return NULL;
  }
  return sel;
} /* muse_lsf_select_arc_line() */

/*----------------------------------------------------------------------------*/
/**
   @brief Read images and combine the arc lines into one pixel table.
   @param aImages     list of arc images
   @param aTrace      table containing the trace solution
   @param aWave       table containing the wavelength calibration solution
   @param aArcLines   arc reference line table
   @param aQuality    minimal line quality to select
   @param aWindow     wavelength window around each line
   @return combined pixel table or NULL on error
   @cpl_ensure{aImages, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aTrace, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aWave, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aArcLines, CPL_ERROR_NULL_INPUT, NULL}

   Create pixel tables from all images in the aImages list, select the pixels
   around each arc line, and store the resulting pixels in a new pixel table.

   The output pixel table will have additional columns
   <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt>, containing the reference wavelength, the
   estimated flux, and the background level of the corresponding line.
 */
/*----------------------------------------------------------------------------*/
muse_pixtable *
muse_lsf_create_arcpixtable(muse_imagelist *aImages,
                            cpl_table *aTrace, cpl_table *aWave,
                            cpl_table *aArcLines, int aQuality, double aWindow)
{
  cpl_ensure(aImages, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aTrace, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aWave, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aArcLines, CPL_ERROR_NULL_INPUT, NULL);

  /* create the output pixel table with two extra columns */
  muse_pixtable *pt = cpl_calloc(1, sizeof(muse_pixtable));
  pt->table = muse_cpltable_new(muse_pixtable_def, 0);
  cpl_table_new_column(pt->table, "line_lambda", CPL_TYPE_FLOAT);
  cpl_table_new_column(pt->table, "line_flux", CPL_TYPE_FLOAT);
  cpl_table_new_column(pt->table, "line_background", CPL_TYPE_FLOAT);

  cpl_size i_image, n_images = muse_imagelist_get_size(aImages);
#if 0 /* no extra levels of parallelization in basic processing recipes */
  #pragma omp parallel for default(none)                 /* as req. by Ralf */ \
          shared(i_image, n_images, aImages, aTrace, aWave, aArcLines,         \
                 aQuality, aWindow, pt)
#endif
  for (i_image = 0; i_image < n_images; i_image++) {
    muse_image *image = muse_imagelist_get(aImages, i_image);
    muse_pixtable *pixtable = muse_pixtable_create(image, aTrace, aWave, NULL);
    if (pixtable == NULL) {
      continue;
    }
    /* create header of output pixel table, if it's not there yet */
    if (!pt->header) {
      pt->header = cpl_propertylist_duplicate(pixtable->header);
    }
    char *l = muse_utils_header_get_lamp_names(pixtable->header, '|');
    if (strlen(l) == 0) {
      cpl_msg_warning(__func__,
                      "Ignoring frame without arc lamp switched on");
      cpl_free(l);
      cpl_free(pixtable);
      continue;
    }
    char *lamp = cpl_malloc(strlen(l)+3);
    sprintf(lamp, "|%s|", l);
    cpl_free(l);
    cpl_size i_line;
    cpl_size n_entries = 0;
    muse_ins_mode insmode = muse_pfits_get_mode(pt->header);
    muse_pixtable **slice_pixtable = muse_pixtable_extracted_get_slices(pixtable);
    int n_slices = muse_pixtable_extracted_get_size(slice_pixtable);
    for (i_line = 0; i_line < cpl_table_get_nrow(aArcLines); i_line++) {
      double lambda = cpl_table_get_float(aArcLines, "lambda", i_line, NULL);
      if (!muse_wave_lines_covered_by_data(lambda, insmode) ||
          cpl_table_get_int(aArcLines, "quality", i_line, NULL) < aQuality) {
        continue;
      }
      const char *n = muse_wave_lines_get_lampname(aArcLines, i_line);
      char *name = cpl_malloc(strlen(n) + 3);
      sprintf(name, "|%s|", n);
      if (strstr(name, lamp)) {
        int i_slice;
#if 0 /* no extra levels of parallelization in basic processing recipes */
        #pragma omp parallel for default(none)           /* as req. by Ralf */ \
                shared(i_slice, n_slices, n_entries, lambda, slice_pixtable,   \
                       aWindow, pt)
#endif
        for (i_slice = 0; i_slice < n_slices; i_slice++) {
          cpl_table *sel = muse_lsf_select_arc_line(slice_pixtable[i_slice],
                                                    lambda, aWindow);
          if (sel) {
#if 0 /* no parallelization here, not necessary any more */
            #pragma omp critical(construct_pixtable)
#endif
            cpl_table_insert(pt->table, sel, cpl_table_get_nrow(pt->table));
#if 0 /* no parallelization here, not necessary any more */
            #pragma omp atomic
#endif
            n_entries += cpl_table_get_nrow(sel);
            cpl_table_delete(sel);
          }
        }
      }
      cpl_free(name);
    } /* for i_line (arc lines) */
    muse_pixtable_extracted_delete(slice_pixtable);
    muse_pixtable_delete(pixtable);
    cpl_msg_info(__func__, "Using %"CPL_SIZE_FORMAT" entries with lamp %s",
                 n_entries, lamp);
    cpl_free(lamp);
  } /* for i_image (all images in list) */
  return pt;
} /* muse_lsf_create_arcpixtable() */

/*----------------------------------------------------------------------------*/
/**
   @private
   @brief Do a polynomial fit on an arc pixel table.
   @param aPixtable      special pixel table with arc lines
   @param aOrderLsf      polynomial order in x (LSF) direction
   @param aOrderLambda   polynomial order in y (arc line wavelength) direction
   @return fitted polynomial, or NULL or error
   @cpl_ensure{aPixtable["d_lambda"], CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_lambda"], CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["data"], CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_flux"], CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_background"], CPL_ERROR_ILLEGAL_INPUT, NULL}

   The fit is done in two dimensions: x direction is the pixel wavelength,
   relative to the arc line (column <tt>d_lambda</tt>),
   and y direction is the arc line wavelength (column <tt>line_lambda</tt>).

   The data are normalized by dividing by the according line flux
   (column <tt>line_flux</tt>) and line background (column
   <tt>line_background</tt>)

   If there are not enough data points, or if the fit fails for other
   reasons, the funtion returns NULL, without setting an error.
 */
/*----------------------------------------------------------------------------*/
static cpl_polynomial *
muse_lsf_fit_polynomial(cpl_table *aPixtable, int aOrderLsf, int aOrderLambda)
{
  cpl_errorstate prestate = cpl_errorstate_get();
  cpl_size nrows = cpl_table_get_nrow(aPixtable);
  if (nrows <= (aOrderLsf + 1) * (aOrderLambda)) {
    return NULL;
  }
  double *p_dlambda = cpl_table_get_data_double(aPixtable, "d_lambda");
  cpl_ensure(p_dlambda, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_line = cpl_table_get_data_float(aPixtable, "line_lambda");
  cpl_ensure(p_line, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_data = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_DATA);
  cpl_ensure(p_data, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_flux = cpl_table_get_data_float(aPixtable, "line_flux");
  cpl_ensure(p_flux, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_background = cpl_table_get_data_float(aPixtable, "line_background");
  cpl_ensure(p_background, CPL_ERROR_ILLEGAL_INPUT, NULL);

  cpl_matrix *coord = cpl_matrix_new(2, nrows);
  cpl_vector *data = cpl_vector_new(nrows);
  double *coord_ptr = cpl_matrix_get_data(coord);
  double *data_ptr = cpl_vector_get_data(data);
  cpl_size i_row;

  for (i_row = 0; i_row < nrows; i_row++) {
    coord_ptr[i_row] = p_dlambda[i_row];
    coord_ptr[i_row + nrows] = p_line[i_row];
    data_ptr[i_row] = (p_data[i_row] - p_background[i_row]) / p_flux[i_row];
  }

  cpl_polynomial *p = cpl_polynomial_new(2);
  const cpl_size maxdeg2d[] = { aOrderLsf, aOrderLambda };
  cpl_polynomial_fit(p, coord, NULL, data, NULL, CPL_TRUE, NULL, maxdeg2d);
  cpl_matrix_delete(coord);
  cpl_vector_delete(data);

  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_errorstate_set(prestate);
    return NULL;
  }

  return p;
} /* muse_lsf_fit_polynomial() */

/*----------------------------------------------------------------------------*/
/**
   @brief Compute the LSF for all wavelengths of one slice.
   @param aPixtable              Special pixel table with arc lines.
   @param aLsfImage              Image to store the LSF.
   @param aWCS                   The WCS of the image.
   @param aLsfRegressionWindow   The Regression window in LSF direction.
   @return CPL_ERROR_NONE if everything went OK or another CPL error code on
           failure
   @cpl_ensure_code{aPixtable, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aLsfImage, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aWCS, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aLsfRegressionWindow > 0, CPL_ERROR_ILLEGAL_INPUT}

   The pixel table is assumed to be restricted to one slice.

   The LSF is computed as a twodimensional regression fit, with the
   wavelength of the pixel in the x (LSF) direction, and the arc line
   reference wavelength in y direction. The regression window in x
   (LSF) direction is taken as parameter; in y (line wavelength)
   direction always the full range is used.

   As fit, a polynomial fit with order 2 in x (LSF) and order 3 in y
   (line wavelength) direction is used. One fit per x value is done.

   After the fit, the according pixels in the image are set to the
   LSF. If a fit did not succeed, the pixels in the according image
   column are set invalid.
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_fit_slice(const muse_pixtable *aPixtable, cpl_image *aLsfImage,
                   muse_wcs *aWCS, double aLsfRegressionWindow)
{
  cpl_ensure_code(aPixtable && aPixtable->table, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aLsfImage, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aLsfRegressionWindow > 0, CPL_ERROR_ILLEGAL_INPUT);

  cpl_size nrows = muse_pixtable_get_nrow(aPixtable);
  if (nrows == 0) {
    return CPL_ERROR_NONE;
  }
  /* work on a locally duplicated table (since we mess with the columns) */
  cpl_table *pixtable = cpl_table_duplicate(aPixtable->table);
  uint32_t origin = cpl_table_get_int(pixtable, MUSE_PIXTABLE_ORIGIN, 0, NULL);
  int i_ifu = muse_pixtable_origin_get_ifu(origin);
  int i_slice = muse_pixtable_origin_get_slice(origin);
  cpl_msg_info(__func__, "processing slice %2i.%02i"
               " with %"CPL_SIZE_FORMAT" entries",
               i_ifu, i_slice, nrows);

  /* Create temporary column with (lmbda - line_lambda) and sort pixel table
     according to this column. Sorting is important since we use it later
     to extract the values according to the regession window using
     find_sorted(). */
  cpl_table_cast_column(pixtable, MUSE_PIXTABLE_LAMBDA, "d_lambda", CPL_TYPE_DOUBLE);
  cpl_table_subtract_columns(pixtable, "d_lambda", "line_lambda");
  cpl_propertylist *order = cpl_propertylist_new();
  cpl_propertylist_append_bool(order, "d_lambda", CPL_FALSE);
  cpl_table_sort(pixtable, order);
  cpl_propertylist_delete(order);

  /* Loop over all x (LSF) data points and create a polynomial for each
     covering the whole wavelength range */
  cpl_size i_lsf;
  cpl_size n_lsf = cpl_image_get_size_x(aLsfImage);
  cpl_size n_lambda = cpl_image_get_size_y(aLsfImage);
  for (i_lsf = 1; i_lsf <= n_lsf; i_lsf++) {
    double x = aWCS->crval1 + (i_lsf - aWCS->crpix1) * aWCS->cd11;
    double x_ref = CPL_MAX(x, cpl_table_get_column_min(pixtable, "d_lambda")
                           + aLsfRegressionWindow);
    x_ref = CPL_MIN(x, cpl_table_get_column_max(pixtable, "d_lambda")
                    - aLsfRegressionWindow);
    cpl_size low = muse_cpltable_find_sorted(pixtable, "d_lambda",
                                             x_ref - aLsfRegressionWindow);
    cpl_size high = muse_cpltable_find_sorted(pixtable, "d_lambda",
                                              x_ref + aLsfRegressionWindow);
    cpl_table *selected = cpl_table_extract(pixtable, low+1, high-low);
    cpl_polynomial *p = muse_lsf_fit_polynomial(selected, 2, 3);
    if (p) {
      cpl_vector *c = cpl_vector_new(2);
      cpl_vector_set(c, 0, x);
      cpl_size i_lambda;
      /* Fill the image column with the values calculated from the polynomial */
      for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
        double y = aWCS->crval2 + (i_lambda - aWCS->crpix2) * aWCS->cd22;
        cpl_vector_set(c, 1, y);
        double flux = cpl_polynomial_eval(p, c);
        cpl_image_set(aLsfImage, i_lsf, i_lambda, flux);
      }
      cpl_vector_delete(c);
    } else {
      cpl_msg_warning(__func__,
                      "Failed polynomial fit %2i.%02i %.2f;"
                      " %"CPL_SIZE_FORMAT" entries",
                      i_ifu, i_slice, x, cpl_table_get_nrow(selected));
      cpl_size i_lambda;
      for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
        cpl_image_reject(aLsfImage, i_lsf, i_lambda);
      }
    }
    cpl_polynomial_delete(p);
    cpl_table_delete(selected);
  }
  cpl_table_delete(pixtable);
  /* Re-normalize total area of each LSF, since normalization may
     not kept during the polynomial fit */
  cpl_size i_lambda;
  for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
    double line_flux = cpl_image_get_flux_window(aLsfImage, 1, i_lambda,
                                                 n_lsf, i_lambda) * aWCS->cd11;
    for (i_lsf = 1; i_lsf <= n_lsf; i_lsf++) {
      int invalid;
      double flux = cpl_image_get(aLsfImage, i_lsf, i_lambda, &invalid);
      if (!invalid) {
        cpl_image_set(aLsfImage, i_lsf, i_lambda, flux / line_flux);
      }
    }
  }
  return CPL_ERROR_NONE;
} /* muse_lsf_fit_slice() */

/*----------------------------------------------------------------------------*/
/**
   @brief Create a new LSF datacube.
   @param aLsfHalfRange   Half width of the range in LSF direction
   @param aNLsf           Number of data points in LSF direction
   @param aNLambda        Number of data points in line wavelength direction
   @param aHeader         Header to derive the LSF cube header from (can be NULL)
   @return The newly created data cube.
 */
/*----------------------------------------------------------------------------*/
muse_lsf_cube *
muse_lsf_cube_new(double aLsfHalfRange, cpl_size aNLsf, cpl_size aNLambda,
                  const cpl_propertylist *aHeader)
{
  muse_lsf_cube *lsf = cpl_calloc(1, sizeof(muse_lsf_cube));
  lsf->header = cpl_propertylist_new();
  if (aHeader) {
    /* copy all keywords, except keys that don't belong here */
    const char *regex = MUSE_HDR_OVSC_REGEXP"|"MUSE_WCS_KEYS"|"MUSE_HDR_PT_REGEXP;
    cpl_propertylist_copy_property_regexp(lsf->header, aHeader, regex, 1);
  }
  lsf->img = cpl_imagelist_new();
  cpl_size i;
  for (i = 0; i < kMuseSlicesPerCCD; i++) {
    cpl_imagelist_set(lsf->img,
                      cpl_image_new(aNLsf, aNLambda, CPL_TYPE_FLOAT), i);
  }
  double lsf_step = 2*aLsfHalfRange / (aNLsf - 1),
         lambda_step = (kMuseNominalLambdaMax - kMuseNominalLambdaMin)
                     / (aNLambda - 1);
  lsf->wcs = cpl_calloc(1, sizeof(muse_wcs));
  lsf->wcs->cd11 = lsf_step;
  lsf->wcs->cd12 = 0.;
  lsf->wcs->cd21 = 0.;
  lsf->wcs->cd22 = lambda_step;
  lsf->wcs->crval1 = -aLsfHalfRange;
  lsf->wcs->crval2 = kMuseNominalLambdaMin;
  lsf->wcs->crpix1 = 1.;
  lsf->wcs->crpix2 = 1.;
  return lsf;
} /* muse_lsf_cube_new() */

/*----------------------------------------------------------------------------*/
/**
   @brief Deallocate the memory for the LSF cube.
   @param aLsfCube The LSF cube to deallocate
 */
/*----------------------------------------------------------------------------*/
void
muse_lsf_cube_delete(muse_lsf_cube *aLsfCube)
{
  if (!aLsfCube) {
    return;
  }
  cpl_propertylist_delete(aLsfCube->header);
  cpl_imagelist_delete(aLsfCube->img);
  cpl_free(aLsfCube->wcs);
  cpl_free(aLsfCube);
} /* muse_lsf_cube_delete() */

/*----------------------------------------------------------------------------*/
/**
   @brief Save the LSF cube to disk.
   @param aLsfCube  The LSF cube
   @param aFileName FITS file name
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_cube_save(muse_lsf_cube *aLsfCube, const char *aFileName)
{
  cpl_ensure_code(aLsfCube, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = cpl_propertylist_save(aLsfCube->header, aFileName,
                                            CPL_IO_CREATE);
  if (rc != CPL_ERROR_NONE) {
    return rc;
  }
  /* create the extension header */
  cpl_propertylist *header = cpl_propertylist_new();
  cpl_propertylist_append_string(header, "EXTNAME", "LSF_PROFILE");
  cpl_propertylist_append_int(header, "WCSAXES", 2);
  cpl_propertylist_append_double(header, "CD1_1", aLsfCube->wcs->cd11);
  cpl_propertylist_append_double(header, "CD1_2", aLsfCube->wcs->cd12);
  cpl_propertylist_append_double(header, "CD2_1", aLsfCube->wcs->cd21);
  cpl_propertylist_append_double(header, "CD2_2", aLsfCube->wcs->cd22);
  cpl_propertylist_append_double(header, "CRPIX1", aLsfCube->wcs->crpix1);
  cpl_propertylist_append_double(header, "CRPIX2", aLsfCube->wcs->crpix2);
  cpl_propertylist_append_double(header, "CRVAL1", aLsfCube->wcs->crval1);
  cpl_propertylist_append_double(header, "CRVAL2", aLsfCube->wcs->crval2);
  cpl_propertylist_append_string(header, "CTYPE1", "PARAM");
  cpl_propertylist_append_string(header, "CTYPE2", "AWAV");
  cpl_propertylist_append_string(header, "CUNIT1", "Angstrom");
  cpl_propertylist_append_string(header, "CUNIT2", "Angstrom");

  rc = cpl_imagelist_save(aLsfCube->img, aFileName,
                          CPL_TYPE_FLOAT, header, CPL_IO_EXTEND);
  cpl_propertylist_delete(header);
  return rc;
} /* muse_lsf_cube_save() */

/*----------------------------------------------------------------------------*/
/**
   @brief Load all LSF cubes for all IFUs into an array.
   @param aProcessing   the processing structure
   @return array of up to kMuseNumIFUs LSF cubes or NULL on error

   This function output a warning message, if one of the files from the input
   frameset cannot be loaded or if the LSFs for one of the IFUs cannot be
   found.

   @cpl_ensure{aProcessing, CPL_ERROR_NULL_INPUT, NULL}
 */
/*----------------------------------------------------------------------------*/
muse_lsf_cube **
muse_lsf_cube_load_all(muse_processing *aProcessing)
{
  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
  muse_lsf_cube **lsf = cpl_calloc(kMuseNumIFUs, sizeof(muse_lsf_cube *));
  unsigned char ifu, nlsfs = 0; /* count number of LSF cubes loaded */
  for (ifu = 1; ifu <= kMuseNumIFUs; ifu++) {
    cpl_frameset *frames = muse_frameset_find(aProcessing->inframes,
                                              MUSE_TAG_LSF_PROFILE, ifu, CPL_FALSE);
    cpl_errorstate es = cpl_errorstate_get();
    cpl_frame *frame = cpl_frameset_get_position(frames, 0);
    if (frame == NULL) {
      cpl_msg_warning(__func__, "No %s (cube format) specified for IFU %2hhu!",
                      MUSE_TAG_LSF_PROFILE, ifu);
      cpl_errorstate_set(es); /* swallow the illegal input state */
      cpl_frameset_delete(frames);
      continue;
    }
    const char *fn = cpl_frame_get_filename(frame);
    lsf[ifu-1] = muse_lsf_cube_load(fn, ifu);
    if (lsf[ifu-1] == NULL) {
      cpl_msg_warning(__func__, "Could not load LSF (cube format) for IFU %2hhu "
                      "from \"%s\"!", ifu, fn);
      cpl_frameset_delete(frames);
      muse_lsf_cube_delete_all(lsf);
      return NULL;
    }
    muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_CALIB, 1);
    cpl_frameset_delete(frames);
    nlsfs++;
  } /* for ifu */
  if (!nlsfs) {
    cpl_msg_error(__func__, "Did not load any %ss (cube format)!",
                  MUSE_TAG_LSF_PROFILE);
    muse_lsf_cube_delete_all(lsf);
    return NULL;
  } /* if none loaded */
  cpl_msg_info(__func__, "Successfully loaded %s%hhu %ss (cube format).",
               nlsfs == kMuseNumIFUs ? "all ": "", nlsfs, MUSE_TAG_LSF_PROFILE);
  return lsf;
} /* muse_lsf_cube_load_all() */

/*----------------------------------------------------------------------------*/
/**
   @brief Delete all LSF cubes.
   @param aLsfCube   array of 48 data cubes
 */
/*----------------------------------------------------------------------------*/
void
muse_lsf_cube_delete_all(muse_lsf_cube **aLsfCube)
{
  if (!aLsfCube) {
   return;
  }
  int ifu;
  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    muse_lsf_cube_delete(aLsfCube[ifu]);
  }
  cpl_free(aLsfCube);
} /* muse_lsf_cube_delete_all() */

/*----------------------------------------------------------------------------*/
/**
   @brief Load a LSF cube for one single IFU from disk.
   @param aFileName   FITS file name
   @param aIFU        the IFU number
   @return the LSF cube.
   @cpl_ensure{aFileName, CPL_ERROR_NULL_INPUT, NULL}
 */
/*----------------------------------------------------------------------------*/
muse_lsf_cube *
muse_lsf_cube_load(const char *aFileName, unsigned char aIFU)
{
  cpl_ensure(aFileName, CPL_ERROR_NULL_INPUT, NULL);

  /* start looking for the LSF_PROFILE extension */
  int ext = cpl_fits_find_extension(aFileName, "LSF_PROFILE");
  char *extname = NULL;
  if (ext <= 0) { /* if not in an extension then look for merged name */
    extname = cpl_sprintf("CHAN%02hhu.LSF_PROFILE", aIFU);
    ext = cpl_fits_find_extension(aFileName, extname);
    if (ext <= 0) { /* no extension for given IFU found */
      cpl_free(extname);
      cpl_error_set(__func__, CPL_ERROR_DATA_NOT_FOUND);
      return NULL;
    }
  } /* if no LSF_PROFILE extension */
  cpl_free(extname);

  muse_lsf_cube *lsf = cpl_calloc(1, sizeof(muse_lsf_cube));
  lsf->header = cpl_propertylist_load(aFileName, 0); /* primary header */
  lsf->img = cpl_imagelist_load(aFileName, CPL_TYPE_DOUBLE, ext);
  lsf->wcs = cpl_calloc(1, sizeof(muse_wcs));
  if (lsf->img == NULL) {
    muse_lsf_cube_delete(lsf);
    return NULL;
  }
  /* now load the WCS from the extension header */
  cpl_propertylist *header = cpl_propertylist_load(aFileName, ext);
  if (header == NULL) {
    muse_lsf_cube_delete(lsf);
    return NULL;
  }
  lsf->wcs->cd11 = muse_pfits_get_cd(header, 1, 1);
  lsf->wcs->cd12 = muse_pfits_get_cd(header, 1, 2);
  lsf->wcs->cd21 = muse_pfits_get_cd(header, 2, 1);
  lsf->wcs->cd22 = muse_pfits_get_cd(header, 2, 2);
  lsf->wcs->crpix1 = muse_pfits_get_crpix(header, 1);
  lsf->wcs->crpix2 = muse_pfits_get_crpix(header, 2);
  lsf->wcs->crval1 = muse_pfits_get_crval(header, 1);
  lsf->wcs->crval2 = muse_pfits_get_crval(header, 2);
  /* XXX check types and units of the WCS */
  // muse_pfits_get_ctype(header, 1) "PARAM"
  // muse_pfits_get_ctype(header, 2) "AWAV"
  // muse_pfits_get_cunit(header, 1) "Angstrom"
  // muse_pfits_get_cunit(header, 2) "Angstrom"
  cpl_propertylist_delete(header);

  return lsf;
} /* muse_lsf_cube_load() */

/*----------------------------------------------------------------------------*/
/**
   @brief Create an average image from all LSF cubes
   @param aLsfCube    array of all LSF cubes
   @param aPixtable   pixel table for the weighting, or NULL
   @return Average LSF image

   All images from all cubes are squeezed into one image. If a
   pixel table is given, each IFU/slice is weighted with the number of
   entries in the pixel table for this IFU/slice.
 */
/*----------------------------------------------------------------------------*/
cpl_image *
muse_lsf_average_cube_all(muse_lsf_cube **aLsfCube, muse_pixtable *aPixtable)
{
  cpl_ensure(aLsfCube, CPL_ERROR_NULL_INPUT, NULL);
  cpl_image *img = NULL;
  cpl_size wsum = 0;
  cpl_size w[kMuseNumIFUs][kMuseSlicesPerCCD];
  cpl_size ifu;
  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    cpl_size i_slice;
    for (i_slice = 0; i_slice < kMuseSlicesPerCCD; i_slice++) {
      w[ifu][i_slice] = (aPixtable == NULL)?1.0:0.0;
    }
  }

  if (aPixtable) {
    cpl_size n_rows = muse_pixtable_get_nrow(aPixtable);
    cpl_size i_row;
    uint32_t *origin = (uint32_t *)cpl_table_get_data_int(aPixtable->table,
                                                          MUSE_PIXTABLE_ORIGIN);
    for (i_row = 0; i_row < n_rows; i_row++) {
      int i_ifu = muse_pixtable_origin_get_ifu(origin[i_row]);
      int i_slice = muse_pixtable_origin_get_slice(origin[i_row]);
      w[i_ifu-1][i_slice-1]++;
    }
  }

  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    if (aLsfCube[ifu] == NULL) {
      continue;
    }
    cpl_size i_slice;
    cpl_size n_slices = cpl_imagelist_get_size(aLsfCube[ifu]->img);
    for (i_slice = 0; i_slice < n_slices; i_slice++) {
      if (w[ifu][i_slice] > 0) {
        cpl_image *m = cpl_imagelist_get(aLsfCube[ifu]->img, i_slice);
        m = cpl_image_duplicate(m);
        cpl_image_multiply_scalar(m, w[ifu][i_slice]);
        wsum += w[ifu][i_slice];
        if (img == NULL) {
          img = m;
        } else {
          cpl_errorstate pre = cpl_errorstate_get();
          cpl_error_code rc = cpl_image_add(img, m);
          cpl_image_delete(m);
          if (rc != CPL_ERROR_NONE) {
            cpl_msg_warning(__func__, "Could not add cube of IFU %"
                            CPL_SIZE_FORMAT": %s",
                            ifu+1, cpl_error_get_message());
            cpl_errorstate_set(pre);
            break;
          }
        }
      }
    }
  }
  if ((img) && (wsum > 0)) {
    cpl_image_divide_scalar(img, wsum);
    return img;
  } else {
    cpl_image_delete(img);
    return NULL;
  }
} /* muse_lsf_average_cube_all() */

/*----------------------------------------------------------------------------*/
/**
   @brief Get the world coordinate representative of all LSF cubes.
   @param aLsfCubes   the array of LSF cubes
   @return a representative WCS or NULL on error

   Since apparently nobody messed with the cube's WCS, it is enough to just
   return the first WCS found.

   XXX this should really check that all WCSs are identical!

   @error{set CPL_ERROR_NULL_INPUT\, return NULL, aLsfCubes are NULL}
 */
/*----------------------------------------------------------------------------*/
muse_wcs *
muse_lsf_cube_get_wcs_all(muse_lsf_cube **aLsfCubes)
{
  cpl_ensure(aLsfCubes, CPL_ERROR_NULL_INPUT, NULL);
  int ifu;
  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    if (aLsfCubes[ifu]) {
      return aLsfCubes[ifu]->wcs;
    }
  }
  return NULL;
} /* muse_lsf_cube_get_wcs_all() */

/*----------------------------------------------------------------------------*/
/**
   @brief Filter an LSF image with a rectangle to model spectrum binning.
   @param aLsfImage   The LSF image. Filtering is in-place.
   @param aWCS        The WCS of the image.
   @param aBinWidth   The width of the rectangle.
   @return CPL_ERROR_NONE on success another CPL error code on failure.

   @error{return CPL_ERROR_NULL_INPUT, aLsfImage and/or aWcs are NULL}
   @error{return CPL_ERROR_ILLEGAL_INPUT, aBinWidth is not positive}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_fold_rectangle(cpl_image *aLsfImage, const muse_wcs *aWCS,
                        double aBinWidth)
{
  cpl_ensure_code(aLsfImage && aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aBinWidth > 0., CPL_ERROR_ILLEGAL_INPUT);

  cpl_size ncol_kernel = aBinWidth / aWCS->cd11;
  ncol_kernel = ((ncol_kernel + 1) / 2) * 2 + 1; /* odd number */
  double r = ncol_kernel - aBinWidth / aWCS->cd11;
  cpl_matrix *kernel = cpl_matrix_new(1, ncol_kernel);
  cpl_matrix_fill(kernel, 1.0);
  cpl_matrix_set(kernel, 0, 0, 1 - r/2);
  cpl_matrix_set(kernel, 0, ncol_kernel-1, 1 - r/2);
  cpl_image *lsfImage0 = cpl_image_duplicate(aLsfImage);
  cpl_image_filter(aLsfImage, lsfImage0, kernel, CPL_FILTER_LINEAR,
                   CPL_BORDER_FILTER);
  cpl_matrix_delete(kernel);
  cpl_image_delete(lsfImage0);

  return CPL_ERROR_NONE;
} /* muse_lsf_fold_rectangle() */

/*----------------------------------------------------------------------------*/
/**
   @brief Apply the LSF to a number of data points of one slice
   @param aLsfImage The slice specific image from the LSF cube
   @param aWCS      The WCS of the image.
   @param aVal     Array with wavelengths [A]. Will be replaced by the
                   normalized LSF data points
   @param aLambda  Line wavelength reference [A].
   @return CPL_ERROR_NONE if everything went OK or another CPL error code on
           failure
   @cpl_ensure_code{aLsfImage, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aWCS, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aVal, CPL_ERROR_NULL_INPUT}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_apply(const cpl_image *aLsfImage, const muse_wcs *aWCS,
               cpl_array *aVal, double aLambda)
{
  cpl_ensure_code(aLsfImage, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aVal, CPL_ERROR_NULL_INPUT);
  cpl_size n_lsf = cpl_image_get_size_x(aLsfImage);
  cpl_size n_lambda = cpl_image_get_size_y(aLsfImage);

  /* Find y position in image corresponding to line wavelength */
  double lambda_pix = (aLambda - aWCS->crval2) / aWCS->cd22 + aWCS->crpix2;
  if (lambda_pix < 1) {
    lambda_pix = 1;
  }
  if (lambda_pix > n_lambda) {
    lambda_pix = n_lambda;
  }
  cpl_size i_lambda = floor(lambda_pix);
  lambda_pix -= i_lambda; /* lambda_pix has fraction of y position */

  /* Create vector containing the LSF by linear interpolation.
     For speed reasons, this is a plain C array. */
  cpl_array *lsf_lambda = cpl_array_new(n_lsf + 4, CPL_TYPE_DOUBLE);
  cpl_array *lsf_values = cpl_array_new(n_lsf + 4, CPL_TYPE_DOUBLE);
  cpl_size i;
  for (i = 1; i <= n_lsf; i++) {
    int res;
    double z = cpl_image_get(aLsfImage, i, i_lambda, &res);
    if (lambda_pix > 0) {
      z = z * (1 - lambda_pix)
        + cpl_image_get(aLsfImage, i, i_lambda+1, &res) * lambda_pix;
    }
    cpl_array_set(lsf_values, i+1, z);
    cpl_array_set(lsf_lambda, i+1, aWCS->crval1 + (i - aWCS->crpix1) * aWCS->cd11);
  }
  // Set outer limits of LSF to zero to avoid invalid values in the fit
  // +- 10000 A is used since the pixel table range will never be in this range
  cpl_array_set(lsf_lambda, 0, -10000);
  cpl_array_set(lsf_values, 0, 0);
  cpl_array_set(lsf_lambda, 1, aWCS->crval1 - aWCS->crpix1 * aWCS->cd11);
  cpl_array_set(lsf_values, 1, 0);
  cpl_array_set(lsf_lambda, n_lsf+2,  aWCS->crval1 + (n_lsf+1 - aWCS->crpix1) * aWCS->cd11);
  cpl_array_set(lsf_values, n_lsf+2, 0);
  cpl_array_set(lsf_lambda, n_lsf+3, 10000);
  cpl_array_set(lsf_values, n_lsf+3, 0);

  // Correct the central position of the LSF function
  cpl_array *v = cpl_array_duplicate(lsf_values);
  cpl_array_multiply(v, lsf_lambda);
  double offset = cpl_array_get_mean(v)/cpl_array_get_mean(lsf_values);
  cpl_array_delete(v);
  cpl_array_subtract_scalar (lsf_lambda, offset);

  // Re-normalize, just to be safe
  double norm = cpl_array_get_mean(lsf_values) * (n_lsf + 4) * aWCS->cd11;
  cpl_array_divide_scalar(lsf_values, norm);

  cpl_array *values = muse_cplarray_interpolate_linear(aVal, lsf_lambda,
                                                       lsf_values);

  // Copy the values back to the function argument
  memcpy(cpl_array_get_data_double(aVal), cpl_array_get_data_double(values),
         cpl_array_get_size(aVal) * sizeof(double));

  cpl_array_delete(values);
  cpl_array_delete(lsf_lambda);
  cpl_array_delete(lsf_values);
  return CPL_ERROR_NONE;
} /* muse_lsf_apply() */

/*----------------------------------------------------------------------------*/
/**
   @brief  Add line spread function to a MUSE slice of a pixel table.
   @param  aPixtable   table containing one slice, with wavelength and
                       an lsf column
   @param  aWavelength line wave length
   @param  aFlux       line flux
   @param  aLsfImage   image containing the line spread function for the slice
   @param  aLsfWCS     WCS definition for aLsfImage

   This will add the LSF for one line to the pixel table. The pixel table is
   required to hold an "lsf" column with values set to zero. Subsequent calls of
   this function add the line spread functions of each call.
*/
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_apply_slice(muse_pixtable *aPixtable, double aWavelength,
                     double aFlux, cpl_image *aLsfImage, muse_wcs *aLsfWCS)
{
  cpl_propertylist *order = cpl_propertylist_new();
  cpl_propertylist_append_bool(order, MUSE_PIXTABLE_LAMBDA, CPL_FALSE);
  cpl_table_sort(aPixtable->table, order);
  cpl_propertylist_delete(order);

  cpl_array *lsf_data = muse_cpltable_extract_column(aPixtable->table, "lsf");

  cpl_array *lambda = NULL;
  if (cpl_table_get_column_type(aPixtable->table, MUSE_PIXTABLE_LAMBDA)
      == CPL_TYPE_DOUBLE) {
    lambda = muse_cpltable_extract_column(aPixtable->table,
                                          MUSE_PIXTABLE_LAMBDA);
  } else {
    cpl_table_cast_column(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                          "lambda_double", CPL_TYPE_DOUBLE);
    lambda = muse_cpltable_extract_column(aPixtable->table, "lambda_double");
  }

  double l_min = aLsfWCS->crval1 + aLsfWCS->cd11 * (1 - aLsfWCS->crpix1),
         l_max = aLsfWCS->crval1
               + aLsfWCS->cd11 * (cpl_image_get_size_x(aLsfImage)
                                  - aLsfWCS->crpix1);
  cpl_size imin = muse_cplarray_find_sorted(lambda, aWavelength + l_min);
  cpl_size imax = muse_cplarray_find_sorted(lambda, aWavelength + l_max);
  cpl_array *l0 = cpl_array_extract(lambda, imin, imax-imin+1);
  cpl_array_subtract_scalar(l0, aWavelength);
  muse_lsf_apply(aLsfImage, aLsfWCS, l0, aWavelength);
  cpl_array_multiply_scalar(l0, aFlux);
  muse_cplarray_add_window(lsf_data, imin, l0);
  cpl_array_delete(l0);
  cpl_array_unwrap(lsf_data);
  cpl_array_unwrap(lambda);
  if (cpl_table_has_column(aPixtable->table, "lambda_double")) {
    cpl_table_erase_column(aPixtable->table, "lambda_double");
  }

  return CPL_ERROR_NONE;
} /* muse_lsf_apply_slice() */

/**@}*/
