From e17815fb1e22bb72a54857cdc6c776db075f4af8 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 2 Jun 2026 14:22:24 +1000 Subject: [PATCH 1/2] Working grid_stat and plotting --- docs/source/reference/workflow/grid_stat.rst | 35 ++++ docs/source/reference/workflow/index.rst | 1 + .../app/grid_stat_plot/bin/grid_stat_plot.py | 197 ++++++++++++++++++ .../app/grid_stat_plot/rose-app.conf | 2 + .../metplus_grid_stat/file/nci-gadi/gpm.conf | 12 ++ .../file/nci-gadi/gridstat.conf | 56 +++++ .../opt/rose-app-nci-gadi.conf | 9 + .../file/{nci => nci-gadi}/pointstat.conf | 36 ++-- .../file/{nci => nci-gadi}/surface.conf | 0 .../opt/rose-app-nci-gadi.conf | 7 +- .../app/metplus_prep_fcst/rose-app.conf | 2 +- src/CSET/cset_workflow/flow.cylc | 1 + .../includes/metplus_common.cylc | 34 +++ .../includes/metplus_grid_stat.cylc | 34 ++- .../includes/metplus_point_stat.cylc | 23 +- src/CSET/operators/_colorbar_definition.json | 15 ++ 16 files changed, 411 insertions(+), 53 deletions(-) create mode 100644 docs/source/reference/workflow/grid_stat.rst create mode 100755 src/CSET/cset_workflow/app/grid_stat_plot/bin/grid_stat_plot.py create mode 100644 src/CSET/cset_workflow/app/grid_stat_plot/rose-app.conf create mode 100644 src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gpm.conf create mode 100644 src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gridstat.conf create mode 100644 src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf rename src/CSET/cset_workflow/app/metplus_point_stat/file/{nci => nci-gadi}/pointstat.conf (51%) rename src/CSET/cset_workflow/app/metplus_point_stat/file/{nci => nci-gadi}/surface.conf (100%) diff --git a/docs/source/reference/workflow/grid_stat.rst b/docs/source/reference/workflow/grid_stat.rst new file mode 100644 index 000000000..797bbccc0 --- /dev/null +++ b/docs/source/reference/workflow/grid_stat.rst @@ -0,0 +1,35 @@ +Comparing against Gridded Data +============================== + +METplus can perform comparison against gridded datasets using the ``grid_stat`` +tool. Enable ``grid_stat`` comparisons by adding to your ``rose-suite.conf``:: + + RUN_METPLUS_GRID_STAT = True + +Observations have to be configured for your site so that METplus knows the +correct source data paths. + +Select which observations to run using e.g.:: + + METPLUS_GRID_STAT_OBS = ["GPM"] + +Output +------ + +The workflow creates spatial plots of the model field, observation field and +difference between model and observation. ``grid_stat`` handles regridding both +data sources to a common grid. + +Observation Types +----------------- + +GPM +^^^ + +https://gpm.nasa.gov/data/imerg + +Compares model field ``stratiform_rainfall_flux`` against observation field +``precipitation_flux``. + +Available sites: +* NCI diff --git a/docs/source/reference/workflow/index.rst b/docs/source/reference/workflow/index.rst index 75244ca9d..633a29343 100644 --- a/docs/source/reference/workflow/index.rst +++ b/docs/source/reference/workflow/index.rst @@ -7,3 +7,4 @@ Reference information for the CSET workflow :maxdepth: 1 observations + grid_stat diff --git a/src/CSET/cset_workflow/app/grid_stat_plot/bin/grid_stat_plot.py b/src/CSET/cset_workflow/app/grid_stat_plot/bin/grid_stat_plot.py new file mode 100755 index 000000000..f8b3838a5 --- /dev/null +++ b/src/CSET/cset_workflow/app/grid_stat_plot/bin/grid_stat_plot.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +"""Create CSET plots from MET grid_stat output.""" + +import argparse +import json +import logging +import os +from pathlib import Path +from typing import Iterable + +import iris.coords +import iris.cube + +from CSET.operators.plot import spatial_pcolormesh_plot + +log = logging.getLogger(__name__) + + +def standardise_names(cube: iris.cube.Cube) -> iris.cube.Cube: + """ + Convert the name MET gives fields to something more standard. + + From a name like + {type}_{var0}_{levels0}(_{var1}_{levels1})?_{region} + + Sets: + var_name to '{type}_{var0}' + long_name to '{var0}_{long_type}' + standard_name to '{var0}' (for OBS and FCST fields only) + + Modifies the cube in-place and returns it + """ + assert cube.var_name is not None + name = cube.var_name + levels = cube.attributes["level"].split(" and ") + mask_region = cube.attributes["masking_region"] + type = name.split("_")[0] + + level_names = [lev.replace("*", "all").replace(",", "_") for lev in levels] + + name_middle = name[len(type) + 1 : -(len(mask_region) + 1)] + + var0 = name_middle[: name_middle.index(level_names[0]) - 1] + + cube.var_name = f"{type}_{var0}_{mask_region}" + cube.attributes["type"] = type + + if type == "OBS": + cube.long_name = f"{var0}_observation" + cube.standard_name = var0 + cube.attributes["verification_type"] = "Observation Value" + elif type == "FCST": + cube.long_name = f"{var0}_forecast" + cube.standard_name = var0 + cube.attributes["verification_type"] = "Forecast Value" + elif type == "DIFF": + cube.long_name = f"{var0}_difference" + cube.attributes["model_var"] = var0 + cube.attributes["verification_type"] = cube.attributes["Difference"] + + return cube + + +def postproc_cube(cube: iris.cube.Cube, field: str, filename: str): + """ + Prepare the MET data for processing. + + Use this as a callback when loading Iris cubes from grid_stat output. + + Fixes the names and sets up time coordinates properly + """ + cube = standardise_names(cube) + + # Grab timestamps + init_time_ut = int(cube.attributes.pop("init_time_ut")) + valid_time_ut = int(cube.attributes.pop("valid_time_ut")) + + # Clean up attributes that might not match between different files + cube.attributes.pop("init_time") + cube.attributes.pop("valid_time") + cube.attributes.pop("level") + cube.attributes.pop("name") + cube.attributes.pop("FileOrigins") + + cube.add_aux_coord( + iris.coords.DimCoord( + init_time_ut, + standard_name="forecast_reference_time", + units="seconds since 1970-01-01 00:00Z", + ) + ) + cube.add_aux_coord( + iris.coords.DimCoord( + valid_time_ut, standard_name="time", units="seconds since 1970-01-01 00:00Z" + ) + ) + + if "invalid_units" in cube.attributes: + cube.units = cube.attributes["invalid_units"].split(" and ")[0] + + +def load_grid_stat(paths: Iterable[Path]) -> iris.cube.CubeList: + """Load processed grid_stat netcdf output as iris cubes.""" + cubes = iris.cube.CubeList() + for path in paths: + cubes.extend(iris.load(path, callback=postproc_cube)) + return cubes.merge() + + +def plot_grid_stat( + cube: iris.cube.Cube, + webdir: Path, + *, + style_file: Path | str | None = None, + plot_resolution: int | None = None, +): + """ + Create plots from grid_stat output. + + Parameters + ---------- + cube: + Input data cube + webdir: + Base web path for this cycle + style_file: + Colorbar definition JSON file + plot_resolution: + Plot resolution in pixels per inch + """ + model = cube.attributes["model"] + obstype = cube.attributes["obtype"] + assert cube.var_name is not None + outdir = webdir / "grid_stat" / f"{model}_vs_{obstype}" / cube.var_name + outdir.mkdir(exist_ok=True, parents=True) + os.chdir(outdir) + + log.info("writing %s to %s", cube.name(), outdir) + + # Set up CSET metadata + meta = { + "category": "Gridded Verification", + "title": f"{cube.long_name}", + "case_date": cube.coord("forecast_reference_time") + .as_string_arrays(fmt="%Y%m%dT%H%MZ") + .points[0], + "MODEL_NAME": model, + "OBSERVATION_TYPE": obstype, + "GRID_STAT_TYPE": cube.attributes["verification_type"], + "SUBAREA_EXTENT": cube.attributes["masking_region"], + } + + if cube.attributes["type"] == "OBS": + meta["title"] = f"{obstype} {cube.long_name}" + elif cube.attributes["type"] == "FCST": + meta["title"] = f"{model} {cube.long_name}" + elif cube.attributes["type"] == "DIFF": + meta["title"] = f"{model} vs {obstype} {cube.long_name}" + + with open("meta.json", "wt") as f: + json.dump(meta, f) + + cube.coord("latitude").guess_bounds() + cube.coord("longitude").guess_bounds() + spatial_pcolormesh_plot(cube, f"grid_stat_DarwinCTL_vs_GPM_{cube.var_name}.png") + + +def main(): + """CLI entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--web-dir", help="directory to store plots", type=Path, required=True + ) + parser.add_argument("--style-file", help="colorbar definitions", type=Path) + parser.add_argument("--plot-resolution", help="plot resolution", type=int) + parser.add_argument( + "input", help="grid_stat directories to process", type=Path, nargs="+" + ) + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + for grid_stat_dir in args.input: + cubes = load_grid_stat(grid_stat_dir.glob("**/*.nc")) + + for cube in cubes: + plot_grid_stat( + cube, + args.web_dir, + style_file=args.style_file, + plot_resolution=args.plot_resolution, + ) + + +if __name__ == "__main__": + main() diff --git a/src/CSET/cset_workflow/app/grid_stat_plot/rose-app.conf b/src/CSET/cset_workflow/app/grid_stat_plot/rose-app.conf new file mode 100644 index 000000000..582ee95f0 --- /dev/null +++ b/src/CSET/cset_workflow/app/grid_stat_plot/rose-app.conf @@ -0,0 +1,2 @@ +[command] +default = grid_stat_plot.py --web-dir "${CYLC_WORKFLOW_SHARE_DIR}/web/plots/${CYLC_TASK_CYCLE_POINT}" "${CYLC_TASK_SHARE_CYCLE_DIR}/grid_stat" diff --git a/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gpm.conf b/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gpm.conf new file mode 100644 index 000000000..76e5efc77 --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gpm.conf @@ -0,0 +1,12 @@ +[config] + +OBTYPE = GPM + +# Daily files contain times 0015 to 2345 +# Make sure valid time 0000 uses the previous day's file by adding a shift of -1s to valid times +OBS_GRID_STAT_INPUT_DIR = /g/data/dp9/da/verification/satellite/products/gpm/netcdf/imerg/NRTlate +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y}/gpm_imerg_NRTlate_V07B_{valid?fmt=%Y%m%d?shift=-1}.nc + +OBS_VAR1_NAME = precipitation_flux +FCST_VAR1_NAME = stratiform_rainfall_flux +BOTH_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" diff --git a/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gridstat.conf b/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gridstat.conf new file mode 100644 index 000000000..bc1817da6 --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_grid_stat/file/nci-gadi/gridstat.conf @@ -0,0 +1,56 @@ +# Common METplus grid_stat configuration to make it work with CSET + +[dir] +MET_INSTALL_DIR = {ENV[CONDA_PREFIX]} + +[config] +PROCESS_LIST = GridStat + +OUTPUT_BASE = {ENV[CYLC_TASK_SHARE_CYCLE_DIR]} +LOG_DIR = {ENV[CYLC_TASK_LOG_DIR]} + +MODEL = {ENV[MODEL_NAME]} + +# [!] Set this for your site/obs type in a site file +# OBTYPE = + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +LOOP_BY = INIT + +INIT_TIME_FMT = %Y%m%dT%H%MZ +INIT_BEG = {ENV[CYLC_TASK_CYCLE_POINT]} +INIT_END = {ENV[CYLC_TASK_CYCLE_POINT]} +INIT_INCREMENT = 1H + +LEAD_SEQ = {ENV[LEAD_SEQ]} + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +FCST_GRID_STAT_INPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/cycle/{init?fmt=%Y%m%dT%H%MZ}/metplus_fcst +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[MODEL_ID]}.nc + +# [!] Set this for your site/obs type in a site file +# OBS_GRID_STAT_INPUT_DIR = +# OBS_GRID_STAT_INPUT_TEMPLATE = + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +GRID_STAT_OUTPUT_TEMPLATE = grid_stat/{MODEL}/{OBTYPE} +GRID_STAT_OUTPUT_PREFIX = {MODEL}_vs_{OBTYPE}_{CURRENT_OBS_NAME} + +# Default regridding +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_METHOD = BILIN +GRID_STAT_REGRID_WIDTH = 2 diff --git a/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf new file mode 100644 index 000000000..482f413db --- /dev/null +++ b/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf @@ -0,0 +1,9 @@ +[command] +default = for TYPE in $GRID_STAT_TYPES; do app_env_wrapper run_metplus.py "$SITE/common.conf" "$SITE/$TYPE.conf"; done + +[env] +!CONDA_VENV_LOCATION=/dev/null +MET_INSTALL_DIR=/bom-ngm/conda + +# TODO: put in top-level +GRID_STAT_TYPES=gpm diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/pointstat.conf similarity index 51% rename from src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf rename to src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/pointstat.conf index 5d7b1880d..2092053e9 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/pointstat.conf @@ -1,24 +1,15 @@ +# Common METplus point_stat configuration to make it work with CSET + [dir] -FCST_BASE = {ENV[METPLUS_FCST_DIR]} -OUTPUT_BASE = {ENV[CYLC_TASK_WORK_DIR]} MET_INSTALL_DIR = {ENV[CONDA_PREFIX]} [config] +PROCESS_LIST = PointStat +OUTPUT_BASE = {ENV[CYLC_TASK_SHARE_CYCLE_DIR]} LOG_DIR = {ENV[CYLC_TASK_LOG_DIR]} -# Documentation for this use case can be found at -# https://metplus.readthedocs.io/en/latest/generated/met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.html - -# For additional information, please see the METplus Users Guide. -# https://metplus.readthedocs.io/en/latest/Users_Guide - -### -# Processes to run -# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list -### - -PROCESS_LIST = PointStat +MODEL = {ENV[MODEL_NAME]} ### # Time Info @@ -32,25 +23,24 @@ PROCESS_LIST = PointStat ### LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%dT%H -INIT_BEG = {ENV[TASK_START_TIME]} -INIT_END = {ENV[TASK_START_TIME]} + +INIT_TIME_FMT = %Y%m%dT%H%MZ +INIT_BEG = {ENV[CYLC_TASK_CYCLE_POINT]} +INIT_END = {ENV[CYLC_TASK_CYCLE_POINT]} INIT_INCREMENT = 1H -LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1) -# Number of seconds to shift times in the fcst file (try half the time increment) -FCST_SHIFT = 1800 +LEAD_SEQ = {ENV[LEAD_SEQ]} ### # File I/O # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info ### -FCST_POINT_STAT_INPUT_DIR = {FCST_BASE} -FCST_POINT_STAT_INPUT_TEMPLATE = m1.nc +FCST_POINT_STAT_INPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/cycle/{init?fmt=%Y%m%dT%H%MZ}/metplus_fcst +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[MODEL_ID]}.nc OBS_POINT_STAT_INPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc OBS_POINT_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc -POINT_STAT_OUTPUT_DIR = {ENV[CYLC_TASK_SHARE_CYCLE_DIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} POINT_STAT_OUTPUT_TEMPLATE = Point_Stat_{ENV[MODEL_NAME]} diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf similarity index 100% rename from src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf rename to src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-nci-gadi.conf index e3432fa7d..7e89a50e2 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-nci-gadi.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/opt/rose-app-nci-gadi.conf @@ -1,5 +1,6 @@ [command] -default=app_env_wrapper run_metplus.py nci/pointstat.conf nci/surface.conf +default=app_env_wrapper run_metplus.py "$SITE/pointstat.conf" "$SITE/surface.conf" -# Don't inherit rose environment -[!env] +[env] +!CONDA_VENV_LOCATION=/dev/null +MET_INSTALL_DIR=/bom-ngm/conda diff --git a/src/CSET/cset_workflow/app/metplus_prep_fcst/rose-app.conf b/src/CSET/cset_workflow/app/metplus_prep_fcst/rose-app.conf index 1d9fccc60..8555782c1 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_fcst/rose-app.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_fcst/rose-app.conf @@ -17,4 +17,4 @@ default = app_env_wrapper cset bake \ = --FILENAME "${MODEL_ID}.nc" [env] -VARNAME = "['air_temperature','relative_humidity']" +VARNAME = "['air_temperature','relative_humidity','stratiform_rainfall_flux']" diff --git a/src/CSET/cset_workflow/flow.cylc b/src/CSET/cset_workflow/flow.cylc index 738518d2a..d60deeaa5 100644 --- a/src/CSET/cset_workflow/flow.cylc +++ b/src/CSET/cset_workflow/flow.cylc @@ -102,6 +102,7 @@ final cycle point = {{CSET_TRIAL_END_DATE}} {% if SKIP_WRITE|default(False) %} SKIP_WRITE = True {% endif %} + SITE = {{SITE}} [[FETCH_DATA]] execution time limit = PT1H diff --git a/src/CSET/cset_workflow/includes/metplus_common.cylc b/src/CSET/cset_workflow/includes/metplus_common.cylc index 665326703..7bfcef5fa 100644 --- a/src/CSET/cset_workflow/includes/metplus_common.cylc +++ b/src/CSET/cset_workflow/includes/metplus_common.cylc @@ -10,4 +10,38 @@ TASK_START_TIME = $(cylc cyclepoint --template CCYYMMDDThh) FORECAST_LENGTH = $(isodatetime --as-total H -- {{ANALYSIS_LENGTH}} | cut -d '.' -f 1 ) TASK_END_TIME = $(cylc cyclepoint --offset {{ANALYSIS_LENGTH}} --template CCYYMMDDThh) + + {% for model in models %} + [[METPLUS_M{{model["id"]}}]] + [[[environment]]] + MODEL_NAME = {{model["name"]}} + model_name = {{model["name"]|lower}} + MODEL_ID = m{{model["id"]}} + # Lead times to analyse as a comma separated list + # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#lead-seq + # Default is hourly from mN_analysis_offset to ANALYSIS_LENGTH + LEAD_SEQ = "{{ + range( + model["analysis_offset"]|duration_as('h')|int, + ANALYSIS_LENGTH|duration_as('h')|int, + 1, + ) | join(',') + }}" + {% endfor %} + + {% for model in models %} + [[metplus_prep_fcst_m{{model["id"]}}]] + # Prepares model data for use by metplus + inherit = METPLUS, METPLUS_M{{model["id"]}} + [[[environment]]] + ROSE_TASK_APP = metplus_prep_fcst + INPUT_PATHS = $CYLC_TASK_SHARE_CYCLE_DIR/data/{{model["id"]}} + {% if SELECT_SUBAREA %} + SUBAREA_TYPE = {{SUBAREA_TYPE}} + SUBAREA_EXTENT = {{SUBAREA_EXTENT}} + {% else %} + SUBAREA_TYPE = None + SUBAREA_EXTENT = None + {% endif %} + {% endfor %} {% endif %} diff --git a/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc b/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc index b3cae4100..d66df9570 100644 --- a/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc @@ -1,12 +1,38 @@ {% if RUN_METPLUS_GRID_STAT|default(False) %} [scheduling] [[graph]] - {{CSET_CYCLE_PERIOD}} = """ - metplus_grid_stat => housekeeping_full - """ + {% macro metplus_point_stat_graph() %} + {% for model in models %} + fetch_fcst_m{{model["id"]}} => metplus_prep_fcst_m{{model["id"]}} + metplus_prep_fcst_m{{model["id"]}} => metplus_grid_stat_m{{model["id"]}} + {% endfor %} + METPLUS_GRID_STAT:succeed-all => grid_stat_plot + grid_stat_plot => cycle_complete + {% endmacro %} + {% if CSET_CYCLING_MODE == "case_study" %} + {% for date in CSET_CASE_DATES %} + R1/{{date}} = """ + {{metplus_point_stat_graph()}} + """ + {% endfor %} + {% elif CSET_CYCLING_MODE == "trial" %} + {{CSET_TRIAL_CYCLE_PERIOD}} = """ + {{metplus_point_stat_graph()}} + """ + {% endif %} [runtime] - [[metplus_grid_stat]] + [[METPLUS_GRID_STAT]] # Runs METplus wrappers for point stat calculations. inherit = METPLUS + [[[environment]]] + ROSE_TASK_APP = metplus_grid_stat + +{% for model in models %} + [[metplus_grid_stat_m{{model["id"]}}]] + inherit = METPLUS_GRID_STAT, METPLUS_M{{model["id"]}} +{% endfor %} + + [[grid_stat_plot]] + {% endif %} diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc index 5f9a9918f..44d0d2dfa 100644 --- a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -35,24 +35,6 @@ [[metplus_prep_obs]] inherit = METPLUS -{% for model in models %} - [[metplus_prep_fcst_m{{model["id"]}}]] - # Prepares model data for use by metplus - inherit = METPLUS - [[[environment]]] - ROSE_TASK_APP = metplus_prep_fcst - MODEL_NAME = {{model["name"]}} - MODEL_ID = m{{model["id"]}} - INPUT_PATHS = $CYLC_TASK_SHARE_CYCLE_DIR/data/{{model["id"]}} - {% if SELECT_SUBAREA %} - SUBAREA_TYPE = {{SUBAREA_TYPE}} - SUBAREA_EXTENT = {{SUBAREA_EXTENT}} - {% else %} - SUBAREA_TYPE = None - SUBAREA_EXTENT = None - {% endif %} -{% endfor %} - [[metplus_ascii2nc]] # Runs METplus wrappers for ASCII to nc ingestion of obs. inherit = METPLUS @@ -80,10 +62,7 @@ {% for model in models %} [[metplus_point_stat_m{{model["id"]}}]] # Runs METplus wrappers for point stat calculations. - inherit = METPLUS_POINT_STAT - [[[environment]]] - MODEL_NAME = {{model["name"]}} - model_name = {{model["name"] | lower}} + inherit = METPLUS_POINT_STAT, METPLUS_M{{model["id"]}} [[metplus_point_stat_postproc_m{{model["id"]}}]] # Archives point stat results in tar files ready for verpy loading. diff --git a/src/CSET/operators/_colorbar_definition.json b/src/CSET/operators/_colorbar_definition.json index 757219d00..72cfe8087 100644 --- a/src/CSET/operators/_colorbar_definition.json +++ b/src/CSET/operators/_colorbar_definition.json @@ -558,6 +558,11 @@ "max": 1e-05, "min": -1e-05 }, + "precipitation_flux": { + "cmap": "cividis", + "max": 50.0, + "min": 0.0 + }, "radar_reflectivity_at_1km_above_the_surface": { "cmap": "cubehelix_r", "max": 70.0, @@ -655,6 +660,16 @@ "max": 6.0, "min": 0.0 }, + "stratiform_rainfall_flux": { + "cmap": "cividis", + "max": 50.0, + "min": 0.0 + }, + "stratiform_rainfall_flux_difference": { + "cmap": "BrBG", + "max": 50.0, + "min": -50.0 + }, "structural_similarity": { "cmap": "afmhot", "max": 1.0, From d18052d3bfaa8258245a014360e21420a4cfbea8 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 2 Jun 2026 16:29:19 +1000 Subject: [PATCH 2/2] Add rose config --- .../app/metplus_grid_stat/opt/rose-app-nci-gadi.conf | 5 +---- .../app/metplus_point_stat/file/nci-gadi/surface.conf | 4 ++-- src/CSET/cset_workflow/includes/metplus_grid_stat.cylc | 1 + src/CSET/cset_workflow/meta/verification/rose-meta.conf | 9 +++++++++ src/CSET/cset_workflow/rose-suite.conf.example | 1 + 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf index 482f413db..433af1f51 100644 --- a/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf +++ b/src/CSET/cset_workflow/app/metplus_grid_stat/opt/rose-app-nci-gadi.conf @@ -1,9 +1,6 @@ [command] -default = for TYPE in $GRID_STAT_TYPES; do app_env_wrapper run_metplus.py "$SITE/common.conf" "$SITE/$TYPE.conf"; done +default = for TYPE in $GRID_STAT_OBS; do app_env_wrapper run_metplus.py "$SITE/gridstat.conf" "$SITE/$TYPE.conf"; done [env] !CONDA_VENV_LOCATION=/dev/null MET_INSTALL_DIR=/bom-ngm/conda - -# TODO: put in top-level -GRID_STAT_TYPES=gpm diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf index d61168298..e1c42c5b4 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci-gadi/surface.conf @@ -17,7 +17,7 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF POINT_STAT_MESSAGE_TYPE = "ADPSFC" FCST_POINT_STAT_VAR1_NAME = air_temperature -FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR1_THRESH = <=273, >273 OBS_POINT_STAT_VAR1_NAME = t2m @@ -25,7 +25,7 @@ OBS_POINT_STAT_VAR1_LEVELS = Z0 OBS_POINT_STAT_VAR1_THRESH = <=273, >273 FCST_POINT_STAT_VAR2_NAME = relative_humidity -FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR2_THRESH = <60, >95 OBS_POINT_STAT_VAR2_NAME = rh2m diff --git a/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc b/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc index d66df9570..45d2829f5 100644 --- a/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_grid_stat.cylc @@ -27,6 +27,7 @@ inherit = METPLUS [[[environment]]] ROSE_TASK_APP = metplus_grid_stat + GRID_STAT_OBS = {{METPLUS_GRID_STAT_OBS|join(" ")}} {% for model in models %} [[metplus_grid_stat_m{{model["id"]}}]] diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 11c6e2901..a4095fc10 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -26,9 +26,18 @@ help=If True, it will enable the production of verification statistics against type=python_boolean trigger=template variables=METPLUS_OPT_CONFIG_KEYS: True; template variables=METPLUS_ANA_DIR: True; + template variables=METPLUS_GRID_STAT_OBS: True; compulsory=true sort-key=met2 +[template variables=METPLUS_GRID_STAT_OBS] +ns=Verification +description=Datasets to run METplus grid stat against. +help=See "app/metplus_grid_stat/file/$SITE" for available options. +type=python_list +compulsory=true +sort-key=met2a + [template variables=METPLUS_OPT_CONFIG_KEYS] ns=Verification description=Which METplus configuration to run. diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 646464bfb..af123cad1 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -84,6 +84,7 @@ MEAN_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False !!METPLUS_BASE="" RUN_METPLUS_GRID_STAT=False RUN_METPLUS_POINT_STAT=False +!!METPLUS_GRID_STAT_OBS=[] !!MET_INSTALL_DIR="" !!MET_LIBRARIES="" !!MLEVEL_TRANSECT_FINISHCOORDS=