Improve NCI observation handling

src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf

@@ -43,16 +43,8 @@ LEAD_SEQ =
 # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info
 ###
 ASCII2NC_INPUT_DIR = {INPUT_BASE}
-ASCII2NC_INPUT_TEMPLATE = *.ascii
+ASCII2NC_INPUT_TEMPLATE = obs.{valid?fmt=%Y%m%dT%H%MZ}.ascii
 ASCII2NC_INPUT_FORMAT = met_point
 
 ASCII2NC_OUTPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc
 ASCII2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc
-
-###
-# ASCII2NC Settings
-# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc
-###
-
-ASCII2NC_WINDOW_BEGIN = 0
-ASCII2NC_WINDOW_END = 0

src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf

@@ -38,9 +38,6 @@ INIT_END = {ENV[TASK_START_TIME]}
 INIT_INCREMENT = 1H
 LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1)
 
-# Number of seconds to shift times in the fcst file (try half the time increment)
-FCST_SHIFT = 1800
-
 ###
 # File I/O
 # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info

src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf

@@ -17,15 +17,15 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF
 POINT_STAT_MESSAGE_TYPE = "ADPSFC"
 
 FCST_POINT_STAT_VAR1_NAME = air_temperature
-FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)"
+FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)"
 FCST_POINT_STAT_VAR1_THRESH = <=273, >273
 
 OBS_POINT_STAT_VAR1_NAME = t2m
 OBS_POINT_STAT_VAR1_LEVELS = Z0
 OBS_POINT_STAT_VAR1_THRESH = <=273, >273
 
 FCST_POINT_STAT_VAR2_NAME = relative_humidity
-FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)"
+FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)"
 FCST_POINT_STAT_VAR2_THRESH =  <60, >95
 
 OBS_POINT_STAT_VAR2_NAME = rh2m

src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py

@@ -7,7 +7,9 @@
 import functools
 import json
 import logging
+import sys
 from abc import ABC, abstractmethod
+from contextlib import nullcontext
 from glob import glob
 from pathlib import Path
 from typing import Iterable, TextIO
@@ -340,13 +342,26 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]:
         """Read ODB2 data."""
         raise NotImplementedError
 
-    def odb2ascii(self, output: TextIO, valid_times: Iterable[TimePoint]):
-        """Write all the observations to a MET ASCII file."""
+    def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]):
+        """
+        Write all the observations to a MET ASCII file.
+
+        If output_pattern contains a strftime-style pattern then the valid time
+        will be used to replace the pattern.
+        """
         for t in valid_times:
+            output = t.strftime(output_pattern)
+
+            if output == "-":
+                out_context = nullcontext(sys.stdout)
+            else:
+                out_context = open(output, "wt")
+
             log.info("Processing %s", t)
-            for obs in self.read_odb(t):
-                ascii = odb2ascii_dataframe(obs)
-                write_ascii(ascii, output)
+            with out_context as f:
+                for obs in self.read_odb(t):
+                    ascii = odb2ascii_dataframe(obs)
+                    write_ascii(ascii, f)
 
 
 class PrepODB2Pattern(PrepODB2):

src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py

@@ -6,17 +6,16 @@
 Valid times can be either ISO timepoints or recurrences, and are used to replace any strftime patterns.
 Data is sourced from the mirror in the ig2 project, not all times are available.
 
-    ./prepODB2.py \
+    ./prepBureauNCI.py \
             --system access-c3-dn \
             --valid-time 20010101T0000Z \
             --valid-time R4/20010102T0000Z/PT6H \
-            --output obs.ascii
+            --output obs.%Y%m%dT%H%MZ.ascii
 """
 
 import argparse
 import logging
 import sys
-from contextlib import nullcontext
 
 from odb2 import valid_times_iterator
 from odb2.bom import BOM_SYSTEMS, PrepBomNci
@@ -56,13 +55,9 @@ def main(argv: list[str]):
         logging.basicConfig(level=logging.INFO)
         sys.tracebacklimit = 0
 
-    if args.output == "-":
-        out_context = nullcontext(sys.stdout)
-    else:
-        out_context = open(args.output, "wt")
-
-    with out_context as output:
-        PrepBomNci(args.system).odb2ascii(output, valid_times_iterator(args.valid_time))
+    PrepBomNci(args.system).odb2ascii(
+        args.output, valid_times_iterator(args.valid_time)
+    )
 
 
 if __name__ == "__main__":

src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py

@@ -9,13 +9,12 @@
     ./prepODB2.py /path/to/%Y/%m/%Y%m%dT%H%MZ/*.odb \
             --valid-time 20010101T0000Z \
             --valid-time R4/20010102T0000Z/PT6H \
-            --output obs.ascii
+            --output obs.%Y%m%dT%H%MZ.ascii
 """
 
 import argparse
 import logging
 import sys
-from contextlib import nullcontext
 
 from odb2.odb2 import PrepODB2Pattern, valid_times_iterator
 
@@ -71,16 +70,10 @@ def main(argv: list[str]):
         # Valid time unset, hopefully the pattern isn't using times
         args.valid_time = ["00010101T0000Z"]
 
-    if args.output == "-":
-        out_context = nullcontext(sys.stdout)
-    else:
-        out_context = open(args.output, "wt")
-
-    with out_context as output:
-        for pattern in args.file:
-            PrepODB2Pattern(pattern).odb2ascii(
-                output, valid_times_iterator(args.valid_time)
-            )
+    for pattern in args.file:
+        PrepODB2Pattern(pattern).odb2ascii(
+            args.output, valid_times_iterator(args.valid_time)
+        )
 
 
 if __name__ == "__main__":

src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf

@@ -1,4 +1,4 @@
 [command]
 # Read in files from ig2
 default = mkdir -p "$METPLUS_OBS_DIR"
-        = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii"
+        = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii"

src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf

@@ -1,4 +1,4 @@
 [command]
 # Read in files from ODB2 files by providing a strftime pattern
 default = mkdir -p "$METPLUS_OBS_DIR"
-        = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" $CUSTOM_ODB2_PATTERN
+        = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" $CUSTOM_ODB2_PATTERN

src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf

@@ -0,0 +1,25 @@
+[template variables]
+SITE="nci-gadi"
+
+# Storage flags to add
+NCI_STORAGE=["gdata/dp9"]
+
+# Where to store output website (view from ARE virtual desktop)
+WEB_DIR="~/public_html/$CYLC_WORKFLOW_NAME"
+
+# Default module
+CSET_ENV_USE_MODULES = True
+CSET_ENV_SEPARATE_MET = False
+MODULES_PURGE = ""
+MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0"
+
+# # Paths to local checkouts of repositories
+# VERPY_DIR = "/path/to/verpy"
+# CSET_DIR = "/path/to/cset"
+
+# # Metplus information
+# # OBS_SYSTEM can be access_g{3,4}, access_c{3,4}_{ad,bn,dn,nq,ph,sy,vt} or custom
+# METPLUS_OBS_SYSTEM = "access_g4"
+#
+# # If OBS_SYSTEM is custom, what path should be read
+# CUSTOM_ODB_PATTERN = "/path/to/files/%Y%m%dT%H%MZ.odb2"

src/CSET/cset_workflow/site/nci-gadi.cylc

@@ -2,10 +2,14 @@
 
 # Site configuration for NCI Gadi
 #
+# Using the opt file 'nci-gadi' is recommended to ensure the CSET module is
+# loaded correctly
+#
 # Important rose-suite.conf settings:
 #   SITE = "nci-gadi"   # Enable the site
 #   PROJECT             # NCI project code
 #   VERPY_DIR           # Path to Verpy source code
+#   NCI_STORAGE         # List of NCI storage locations to use
 #   METPLUS_OBS_SYSTEM  # Where to source obs from
 #                       # Can be a Bureau forecast system e.g.
 #                       # 'access_c3_dn' or 'custom'
@@ -14,16 +18,19 @@
 #
 # Leave METPLUS_FCST_DIR and METPLUS_OBS_DIR unset
 
+{% set STORAGE = [
+    'scratch/'~PROJECT,
+    'gdata/'~PROJECT,
+    'gdata/access',
+    'gdata/hr22',
+] + (NCI_STORAGE | default([])) %}
+
 [runtime]
     [[root]]
         platform = gadi_background
-        init-script = """
-            module use /g/data/access/ngm/modules
-            module load cset/26.4.0
-            """
         [[[ environment ]]]
             PROJECT = {{ PROJECT }}
-            PYTHONPATH = "{{VERPY_DIR}}"
+            PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}/src{% endif %}
 
 {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %}
     [[METPLUS]]
@@ -34,14 +41,14 @@
             -q          = normal
             -l ncpus    = 1
             -l mem      = 4gb
-            -l storage  = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/ig2+scratch/dx2
+            -l storage  = {{ STORAGE | join('+') }}+gdata/ig2
             -W umask    = 0022
         [[[ environment ]]]
             ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}}
 
     [[metplus_prep_obs]]
         [[[ environment ]]]
-            OBS_TIMES = R{{ANALYSIS_LENGTH | duration_as('h') | int }}/$CYLC_TASK_CYCLE_POINT/PT1H
+            OBS_TIMES = R{{(ANALYSIS_LENGTH | duration_as('h') | int) + 1}}/$CYLC_TASK_CYCLE_POINT/PT1H
         {% if METPLUS_OBS_SYSTEM != "custom" %}
             # Forecast system to pull obs from (e.g. 'access_c3_dn')
             METPLUS_OBS_SYSTEM = {{METPLUS_OBS_SYSTEM}}
@@ -60,7 +67,7 @@
             -q          = normal
             -l ncpus    = 48
             -l mem      = 180gb
-            -l storage  = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2
+            -l storage  = {{ STORAGE | join('+') }}
             -W umask    = 0022
         [[[ environment ]]]
             BUNCH_POOL_SIZE = $PBS_NCPUS
@@ -73,7 +80,7 @@
             -q          = normal
             -l ncpus    = 48
             -l mem      = 180gb
-            -l storage  = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2
+            -l storage  = {{ STORAGE | join('+') }}
             -W umask    = 0022
         [[[ environment ]]]
             BUNCH_POOL_SIZE = $PBS_NCPUS