/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2017-2018 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#define _DEFAULT_SOURCE
#define _BSD_SOURCE /* get setenv() from stdlib.h */
#include <stdlib.h> /* setenv() */
#include <string.h> /* strncmp() */

#include <muse.h>

/*----------------------------------------------------------------------------*/
/**
  @brief   Test program to check that function(s) from muse_autocalib
           work as expected when called with the necessary data.

  This program explicitely tests
    muse_autocalib_create_mask
    muse_autocalib_slice_median
    muse_autocalib_apply
 */
/*----------------------------------------------------------------------------*/
int main(int argc, char **argv)
{
  cpl_test_init(PACKAGE_BUGREPORT, CPL_MSG_DEBUG);

  /* testing of muse_resampling_cube*() */
  /* load pixel table, it has spatial units in pixels */
  muse_pixtable *ptpix = NULL;
  if (argc > 1) {
    /* assume that the 1st parameter is the filename of a pixel table */
    ptpix = muse_pixtable_load(argv[1]);
  } else {
    ptpix = muse_pixtable_load(BASEFILENAME"_PIXTABLE_HUDF_short.fits");
  }
  if (!ptpix) {
    return cpl_test_end(1);
  }

  /* reconstruct image from this original data */
  muse_table *fwhite = muse_table_load_filter(NULL, "white");
  muse_resampling_params *p = muse_resampling_params_new(MUSE_RESAMPLE_WEIGHTED_DRIZZLE);
  p->crtype = MUSE_RESAMPLING_CRSTATS_MEDIAN;
  p->crsigma = 5.;
  muse_datacube *cube = muse_resampling_cube(ptpix, p, NULL);
  muse_image *white1 = muse_datacube_collapse(cube, fwhite);
  muse_datacube_delete(cube);

  /* create sky mask and test muse_autocalib_create_mask() */
  cpl_errorstate state = cpl_errorstate_get();
  muse_mask *skymask = muse_autocalib_create_mask(white1, 5.0, "TEST AUTOCAL");
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_nonnull(skymask);
  cpl_table_select_all(ptpix->table);
  muse_pixtable_and_selected_mask(ptpix, skymask, NULL, NULL);
  /* duplicate table for later tests, including the row selection */
  muse_pixtable *pt2 = muse_pixtable_duplicate(ptpix),
                *pt3 = muse_pixtable_duplicate(ptpix);

  /* everything in place, so run the autocalibration now */
  state = cpl_errorstate_get();
  cpl_error_code rc = muse_autocalib_slice_median(ptpix, 15.0, NULL);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test(cpl_propertylist_has(ptpix->header, MUSE_HDR_PT_AUTOCAL_NAME));
  const char *str = cpl_propertylist_get_string(ptpix->header,
                                                MUSE_HDR_PT_AUTOCAL_NAME);
  cpl_msg_debug(__func__, "autocalib string: %s", str);
  cpl_test_zero(strncmp(str, "slice-median", 13));
  str = cpl_propertylist_get_comment(ptpix->header, MUSE_HDR_PT_AUTOCAL_NAME);
  cpl_test_rel(atof(str), 15.0, FLT_EPSILON); /* still the old clip value */

  cpl_table_select_all(ptpix->table);
  cube = muse_resampling_cube(ptpix, p, NULL);
  muse_image *white2 = muse_datacube_collapse(cube, fwhite);
  muse_datacube_delete(cube);

  /* now compare original and corrected image */
#if 0
  muse_image_save(white1, "white1.fits");
  muse_image_save(white2, "white2.fits");
#endif
  muse_image_reject_from_dq(white1);
  muse_image_reject_from_dq(white2);
  cpl_stats *s1 = cpl_stats_new_from_image(white1->data, CPL_STATS_ALL),
            *s2 = cpl_stats_new_from_image(white2->data, CPL_STATS_ALL);
  /* mean and median should change very little */
  cpl_test_rel(cpl_stats_get_mean(s1), cpl_stats_get_mean(s2), 0.1);
  cpl_test_rel(cpl_stats_get_median(s1), cpl_stats_get_median(s2), 0.1);
  /* all deviation should have significantly decreased */
  cpl_test((cpl_stats_get_stdev(s1) / 1.02) > cpl_stats_get_stdev(s2));
  cpl_test((cpl_stats_get_median_dev(s1) / 1.3) > cpl_stats_get_median_dev(s2));
  cpl_test((cpl_stats_get_mad(s1) / 1.7) > cpl_stats_get_mad(s2));
  printf("%s statistics before autocalibration:\n", MUSE_TAG_IMAGE_FOV);
  cpl_stats_dump(s1, CPL_STATS_ALL, stdout);
  printf("%s statistics after autocalibration:\n", MUSE_TAG_IMAGE_FOV);
  cpl_stats_dump(s2, CPL_STATS_ALL, stdout);
  fflush(stdout);
  double median = cpl_stats_get_median(s2);
  cpl_msg_debug(__func__, "global median: %.3f", median);
  cpl_stats_delete(s1);
  cpl_stats_delete(s2);

  /* check the median values in the middle-left slicer stack */
  double m1a = cpl_image_get_median_window(white1->data, 80,  4, 150, 13),
         m1b = cpl_image_get_median_window(white1->data, 80, 16, 150, 25),
         m1c = cpl_image_get_median_window(white1->data, 80, 29, 150, 38),
         m2a = cpl_image_get_median_window(white2->data, 80,  4, 150, 13),
         m2b = cpl_image_get_median_window(white2->data, 80, 16, 150, 25),
         m2c = cpl_image_get_median_window(white2->data, 80, 29, 150, 38);
  cpl_msg_debug(__func__, "median before: %.3f %.3f %.3f", m1a, m1b, m1c);
  cpl_msg_debug(__func__, "median after:  %.3f %.3f %.3f", m2a, m2b, m2c);
  cpl_msg_debug(__func__, "median ratio a: %.5f", m2a / m1a);
  cpl_msg_debug(__func__, "median ratio b: %.5f", m2b / m1b);
  cpl_msg_debug(__func__, "median ratio c: %.5f", m1c / m2c);
  /* individual median values before should be significiantly *
   * different to the global median of after the correction   */
  cpl_test_rel(median - m1a, 0.390, 0.01);
  cpl_test_rel(median - m1b, 0.270, 0.01);
  cpl_test_rel(median - m1c, -0.348, 0.01);
  /* individual median values afterwards should be close to the global median */
  cpl_test_abs(median, m2a, 0.018);
  cpl_test_abs(median, m2b, 0.012);
  cpl_test_abs(median, m2c, 0.040);
  /* the individual median values should have changed less than 1% */
  cpl_test((m2a / m1a) < 1.01 && (m2a / m1a) > 1.);
  cpl_test((m2b / m1b) < 1.01 && (m2b / m1b) > 1.);
  cpl_test((m1c / m2c) < 1.01 && (m1c / m2c) > 1.);

  /* run autocalib again, on the same pixel table, to check that it returns   *
   * without doing anything; use factors table for that: it should still NULL */
  muse_table *factors = NULL;
  state = cpl_errorstate_get();
  rc = muse_autocalib_slice_median(ptpix, 99.9, &factors);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_null(factors);
  str = cpl_propertylist_get_comment(ptpix->header, MUSE_HDR_PT_AUTOCAL_NAME);
  cpl_test_rel(atof(str), 15.0, FLT_EPSILON); /* still the old clip value */

  /* run autocalib again, with factors output and debug messages this *
   * time; manipulate the header to pretend that this is AO data      */
  cpl_propertylist_update_string(pt2->header, "ESO INS MODE", "WFM-AO-N");
  state = cpl_errorstate_get();
  setenv("MUSE_DEBUG_AUTOCALIB", "1", 1);
  rc = muse_autocalib_slice_median(pt2, 15.0, &factors);
  unsetenv("MUSE_DEBUG_AUTOCALIB");
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(factors);
  if (factors) {
    cpl_test_nonnull(factors->table);
    rc = muse_cpltable_check(factors->table, muse_autocal_results_def);
    cpl_test(rc == CPL_ERROR_NONE);
  }
#if 0
  muse_table_save(factors, "factors.fits");
#endif

  /* use the valid factors table to also test muse_autocalib_apply() */
  rc = muse_autocalib_apply(pt2, factors);
  /* should return unchanged, since it was corrected already */
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  str = cpl_propertylist_get_string(pt2->header, MUSE_HDR_PT_AUTOCAL_NAME);
  cpl_test_zero(strncmp(str, "slice-median", 13)); /* still the same... */
  /* now for real, with uncorrected pixel table */
  muse_pixtable *pt4 = muse_pixtable_duplicate(pt3);
  state = cpl_errorstate_get();
  rc = muse_autocalib_apply(pt3, factors);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  str = cpl_propertylist_get_string(pt3->header, MUSE_HDR_PT_AUTOCAL_NAME);
  cpl_test_zero(strncmp(str, "user", 5)); /* it did happen! */
  /* should give an error */
  cpl_table_name_column(factors->table, "corr", "corr_bad");
  state = cpl_errorstate_get();
  rc = muse_autocalib_apply(pt4, factors);
  cpl_test(rc == CPL_ERROR_DATA_NOT_FOUND && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  cpl_test(!cpl_propertylist_has(pt4->header, MUSE_HDR_PT_AUTOCAL_NAME));
  /* should show a warning (but do the work) */
  cpl_size nrow = cpl_table_get_nrow(factors->table);
  cpl_table_name_column(factors->table, "corr_bad", "corr");
  cpl_table_set_size(factors->table, nrow+10);
  state = cpl_errorstate_get();
  rc = muse_autocalib_apply(pt4, factors);
  cpl_test(rc == CPL_ERROR_NONE && cpl_errorstate_is_equal(state));
  cpl_test_nonnull(cpl_propertylist_get_string(pt4->header,
                                               MUSE_HDR_PT_AUTOCAL_NAME));
  muse_pixtable_delete(pt4);
  cpl_table_set_size(factors->table, nrow);

  /* both pt2 and pt3 should have received the exact same correction */
  cpl_array *pt2d = muse_cpltable_extract_column(pt2->table, MUSE_PIXTABLE_DATA),
            *pt2s = muse_cpltable_extract_column(pt2->table, MUSE_PIXTABLE_STAT),
            *pt3d = muse_cpltable_extract_column(pt2->table, MUSE_PIXTABLE_DATA),
            *pt3s = muse_cpltable_extract_column(pt2->table, MUSE_PIXTABLE_STAT);
  cpl_test_array_abs(pt2d, pt3d, DBL_EPSILON);
  cpl_test_array_abs(pt2s, pt3s, DBL_EPSILON);
  cpl_array_unwrap(pt2d);
  cpl_array_unwrap(pt2s);
  cpl_array_unwrap(pt3d);
  cpl_array_unwrap(pt3s);

  cpl_table_select_all(pt2->table);
  cube = muse_resampling_cube(pt2, p, NULL);
  muse_resampling_params_delete(p);
  muse_image *white3 = muse_datacube_collapse(cube, fwhite);
  muse_datacube_delete(cube);
#if 0
  muse_image_save(white3, "white3.fits");
#endif

  /* the AO setting should not matter for this test, since *
   * we cut the wavelength range to exclude it anyway!     */
  cpl_test_image_abs(white2->data, white3->data, FLT_EPSILON);
  muse_image_delete(white2);
  muse_image_delete(white3);
  muse_table_delete(fwhite);
  /* save the header length before deleting */
  int n = cpl_propertylist_get_size(skymask->header);
  muse_mask_delete(skymask);

  /* test failure cases */
  state = cpl_errorstate_get();
  rc = muse_autocalib_slice_median(NULL, 15.0, NULL);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* failure cases of muse_autocal_apply() */
  state = cpl_errorstate_get();
  rc = muse_autocalib_apply(NULL, factors);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  rc = muse_autocalib_apply(pt3, NULL);
  cpl_test(rc == CPL_ERROR_NULL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* factors table with empty header */
  cpl_propertylist_erase_regexp(factors->header, ".*", 0);
  cpl_propertylist_erase_regexp(pt3->header, MUSE_HDR_PT_AUTOCAL_NAME, 0);
  rc = muse_autocalib_apply(pt3, factors);
  cpl_test(rc == CPL_ERROR_ILLEGAL_INPUT && !cpl_errorstate_is_equal(state));
  cpl_errorstate_set(state);
  /* failure cases of muse_autocalib_create_mask() */
  skymask = muse_autocalib_create_mask(NULL, 5.0, "TEST AUTOCAL");
  cpl_test(!cpl_errorstate_is_equal(state) &&
           cpl_error_get_code() == CPL_ERROR_NULL_INPUT);
  cpl_errorstate_set(state);
  cpl_test_null(skymask);
  /* this should work without errors, the output *
   * just doesn't have the new (QC) headers      */
  skymask = muse_autocalib_create_mask(white1, 5.0, NULL);
  cpl_test(cpl_errorstate_is_equal(state));
  cpl_test_nonnull(skymask);
  cpl_test_eq(n - 2, cpl_propertylist_get_size(skymask->header));
  muse_mask_delete(skymask);

  muse_image_delete(white1);
  muse_pixtable_delete(ptpix);
  muse_pixtable_delete(pt2);
  muse_pixtable_delete(pt3);
  muse_table_delete(factors);
  return cpl_test_end(0);
}
