Settings file example

The settings file uses the YAML language, see also this YAML syntax reference.

This file contains all the settings for the paths, recipe parameters, logging, etc.

# NOTE: Top-level keys are substitued in the file before loading it.

# Paths (working directory, database, etc.)
workdir: '.'
db: 'musered.db'
# For Postgres and other remote databases, this allows to specify and
# environment variable with the database url (with the SQLAlchemy specification)
# db_env: 'MXDF_DB'
raw_path: '{workdir}/raw'
reduced_path: '{workdir}/reduced/{version}'
muse_calib_path: '~/lib/musedrs/muse-calib'

# Version
version: '0.1'
revision: '1'

# log level for the terminal output
loglevel: info

# List of additional FITS keywords to get from the raw files (and
# stored in the raw table)
additional_keywords:
  - TELESCOP
  - ESO OBS OBSERVER

# Definition of datasets.
datasets:
  IC4406:
    # This can be used to gather files with different OBJECT names, or to use
    # another name, by default it supposes that the dataset name matches the
    # OBJECT keyword.
    # OBJECT: IC4406
    # OBJECT: [IC4406, IC4406-something]

    # archive_filter contains the parameters passed to
    # astroquery.Eso.query_instrument (its "column_filters" argument).
    # Here for the example we make a query on the target name (obs_targ_name)
    # but one can also use the program id (prog_id) or other query parameters.
    archive_filter:
      obs_targ_name: IC4406

# python-cpl settings, esorex like.
cpl:
  # recipes directory (and version if needed), usually not needed
  # if the DRS is installed in /usr
  recipe_path: '~/.local/lib/esopipes-plugins'
  # to use a specific DRS version:
  # drs_version: '2.1.0'
  # log files directory
  log_dir: '{workdir}/reduced/{version}/logs'
  # severity level and format of messages printed to the terminal
  msg: warning
  msg_format: "[%(process)s] %(levelname)s - %(name)s: %(message)s"
  # msg_format: "[%(process)s] %(levelname)s - %(name)s: %(message)s"
  # severity level and format of messages for the logfile
  esorex_msg: debug
  esorex_msg_format: "%(asctime)s [%(levelname)07s][%(threadName)s] %(name)s: %(message)s"

# This is just an example of how to use substitutions to define date ranges
# that are used below for static calibrations. runs are also used to compute
# a run column in the datase, which allows to select exposures by run.
runs:
  GTO17: &GTO17
    start_date: 2017-04-01
    end_date: 2017-06-30
  GTO19: &GTO19
    start_date: 2017-09-15
    end_date: 2017-09-30

# GTO-only: import databases with comments and flags added by observers
GTO_logs:
  db:
    - "{workdir}/db_logs/GTO-17-P99.db"

# Define additional flags
flags:
  MY_FLAG: "this is a super useful flag"

# Static calibrations.
# By default files from the muse_calib_path directory are used, but it is also
# possible here to specify date ranges of validity for specific calibrations.
static_calib:
  BADPIX_TABLE:
    badpix_table.fits:
      start_date: 2015-05-11
    badpix_table_pre_2015apr.fits:
      start_date: 2014-09-17
      end_date: 2015-05-10
  ASTROMETRY_WCS:
    astrometry_wcs_wfm_gto17.fits: *GTO17
    astrometry_wcs_wfm_gto19.fits: *GTO19
  GEOMETRY_TABLE:
    geometry_table_wfm_gto17.fits: *GTO17
    geometry_table_wfm_gto19.fits: *GTO19

# Parameters for the retrieve_data command (passed to astroquery.Eso.login)
retrieve_data:
  username: MUSE-GTO
  store_password: False

# Global exclusion of frames. The key/value can be either:
# - a DPR_TYPE and exposures names
# - or "raw" and a dict defining a query on the raw table
frames:
  exclude:
    raw:
      - night: ["2017-06-13"]
    BIAS:
      - "2017-06-14T09:01:03"
    MASTER_BIAS:
      - "2017-06-14T09:01:03"

# Parameters for recipes. Can be used to specify additional parameters.
recipes:

  muse_wavecal:
    # This can be used to change the frames excluded by default
    # frames, or to specify a file for a given frame
    #   exclude: MASTER_DARK
    #   include: [MASTER_FLAT]
    params:
      saveimages: True

  muse_scibasic:
    init:
      # all recipes have an output_dir by default, but it can be modified
      output_dir: 'scibasic'

  # Example of a special version of scipost to produce images for the
  # recentering step. For this we don't need sky-subtraction nor raman
  # correction. Can be run with --params muse_scipost_rec
  muse_scipost_rec:
    frames:
      exclude: [RAMAN_LINES]
    init:
      output_dir: 'scipost-not-recentered'
    params:
      dlambda: 5
      # autocalib: 'deepfield'
      skymethod: 'none'
      save: 'cube,autocal'

  muse_exp_align:
    # Compute offsets with the DRS.
    # from_recipe allows to specify the recipe from which files are used.
    from_recipe: muse_scipost_rec
    filt: white

  imphot:
    # Compute offsets with the Imphot.
    from_recipe: muse_scipost_rec
    params:
      fix_beta: 2.5
      hst_filters: ["F775W", "F814W"] # "F606W", , "F850LP"
      # hst_filters_dir: '{workdir}/reduced/{version}/hst_filters'
      # hst_outdir: '{workdir}/reduced/{version}/imphot-hst-images'
      hst_img_dir: '{workdir}/HST/XUDF'
      hst_basename: 'hlsp_xdf_hst_acswfc-30mas_hudf_%s_v1_sci.fits'
      hst_resample_each: True

  muse_scipost:
    # scipost using offsets (OFFSET_LIST computed with the drs method, or a
    # file computed by other means) and other options (raman, skysub, autocalib)
    frames:
      # OFFSET_LIST: OFFSET_LIST_drs
      OFFSET_LIST: '{workdir}/OFFSET_LIST_new.fits'
    init:
      output_dir: 'scipost'
    params:
      # autocalib: 'deepfield'
      skymethod: 'model'
      skymodel_fraction: 0.2
      save: 'cube,skymodel,individual,raman,autocal'

  muse_exp_combine:
    from_recipe: muse_scipost
    # This is with the drs method, where one can specify an OFFSET_LIST
    # frame (see above). Other option is to use MPDAF, see below.
    frames:
      # OFFSET_LIST: drs
      OFFSET_LIST: '{workdir}/OFFSET_LIST_new.fits'

  muse_scipost_make_cube:
    # Example of how to produce cubes with the same OUTPUT_WCS, with
    # process-exp --makecube (this is a special version of muse_scipost just to
    # produce cubes).
    from_recipe: muse_scipost
    frames:
      # OFFSET_LIST can be set either with a string OFFSET_LIST_[method], where
      # method is the one used in muse_exp_align, or directly with a filename
      # OFFSET_LIST: OFFSET_LIST_drs
      OFFSET_LIST: '{workdir}/OFFSET_LIST_new.fits'

      # The OUTPUT_WCS used here was computed by running first muse_exp_combine
      # with method: drs (it is the datacube combined with the DRS).
      OUTPUT_WCS: '{workdir}/reduced/0.1/exp_combine/IC4406/DATACUBE_FINAL.fits'
    init:
      output_dir: 'scipost_cubes'

  mpdaf_combine:
    # Example of cube combination with MPDAF. For this we need cubes that have
    # been produced on the same grid (same OUTPUT_WCS), which is achieved here
    # with muse_scipost_make_cube.
    from_recipe: muse_scipost_make_cube
    params:
      method: sigclip
      version: 0.1

  zap:
    from_recipe: mpdaf_combine
    # frames:
    #   # here we can specify the sources mask
    #   SOURCE_MASK: '{workdir}/MASK.fits'
    init:
      output_dir: 'zap'
    params:
      cftype: median
      cfwidthSVD: 300
      cfwidthSP: 300
      mask_edges: False