Source code for ctdcast.reports.index

"""Tier-3: entry point — generates index, stations list, and sections list pages."""

from __future__ import annotations

import contextlib
import dataclasses
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter
from typing import Any

import numpy as np
import xarray as xr

from ctdcast._version import __version__ as _VERSION
from ctdcast.analysis.bathymetry import interpolate_bathy_at_casts
from ctdcast.analysis.derive import derive_AOU as add_aou
from ctdcast.analysis.derive import derive_teos10_profiles as add_teos10_profiles
from ctdcast.analysis.derive import fill_oxygen_composite
from ctdcast.analysis.geometry import distance_from_km
from ctdcast.config.global_attrs import cruise_name
from ctdcast.config.loader import SectionsConfig, warn_section_yaml_deprecated
from ctdcast.config.parameters import (
    SECTION_BIOGEO_VARS,
    SECTION_PHYSICS_VARS,
    UNKNOWN_CRUISE_ID,
    color_limit_override,
    resolve_sensor_var,
    vlabel,
    vlabel_html,
)
from ctdcast.config.report_config import DEFAULT_REPORT_CONFIG, ReportConfig
from ctdcast.identity import (
    cast_id_from_name,
    compact_cast_list,
    expand_cast_numbers,
    format_cast_id,
)
from ctdcast.processors.fit import FITS_FILENAME
from ctdcast.processors.stage_layout import select_best_available
from ctdcast.readers.ladcp import find_ladcp_file
from ctdcast.reports import _figdebug
from ctdcast.reports._env import get_template
from ctdcast.reports._format import report_generated_at
from ctdcast.reports._navs import (
    CORRECTIONS_TS_HREF,
    inventory_pills,
    inventory_sources,
    nav_for,
)
from ctdcast.reports._plots import (
    RenderedPanel,
    _make_all_sections_map_b64,
    _make_cruise_map_b64,
    _make_ladcp_overview_b64,
    _make_overview_panel_b64,
    _make_section_ts_histogram_b64,
    _make_station_map_b64,  # noqa: F401 — kept for backward compat
    _slot_width,
)
from ctdcast.reports._vendored import nc_tables
from ctdcast.reports._vendored.page_model import (
    Panel,
    PanelGroup,
    Profile,
    ResolvedReport,
    Section,
    resolve,
)
from ctdcast.reports.cast import generate_station_page
from ctdcast.reports.design import _JS_TOP_LINKS, SHARED_CSS, is_qc
from ctdcast.reports.section import generate_section_page
from ctdcast.reports.timeseries import generate_timeseries_page

#: Display slots of the fixed-width cruise figures.  Each is drawn at its slot's
#: width (``_slot_width``) and shown in the same slot.
_MAP_SLOT = "half"
_TS_SLOT = "third"
_OVERVIEW_SLOT = "full"
_CLOCK_SLOT = "full"


# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------


__all__ = ["report"]


[docs] def report( nc_dir: Path, out_dir: Path, *, profiles_path: Path | None = None, groupings_yaml: Path | None = None, section_yaml: Path | None = None, ladcp_dir: Path | None = None, ladcp_profiles_path: Path | None = None, ladcp_pattern: str | None = None, ship_track_nc: Path | None = None, generate: dict[str, bool] | None = None, force: bool = False, skip_existing: bool = False, section_style: str = "pcolormesh", timeseries_style: str = "pcolormesh", vmin_override: dict[str, float] | None = None, vmax_override: dict[str, float] | None = None, cruise_info: dict[str, Any] | None = None, config: ReportConfig | None = None, cast_filter: int | list[int] | None = None, sal_range: tuple[float, float] | None = None, trim_soak: bool = False, dbar_step: int = 1, drop_stub: bool = False, generated_at: str | None = None, ) -> int: """Generate the full ctdcast HTML report suite. Parameters ---------- nc_dir: Directory containing per-cast ``.nc`` files. out_dir: Root output directory. profiles_path: Path to compiled ``profiles.nc``. Required for section and time series pages. groupings_yaml: Cast groupings — ``sections:`` and ``timeseries:``. Conventionally ``ctd_groupings.yaml``. section_yaml: Superseded spelling of *groupings_yaml*; accepted, and used only when *groupings_yaml* is absent. ladcp_dir: Directory containing processed LADCP ``.mat`` files named ``NNN.mat`` or ``NNNb.mat`` (letter-suffix variants supported). ladcp_profiles_path: Path to compiled ``ladcp_profiles.nc``. When present, the summary page gains a Velocity section (cruise-wide U/V overview panels) and a data-inventory pill for the file. ladcp_pattern: Optional filename glob for non-standard LADCP naming, e.g. ``"msm_142_1_*.mat"``. The ``*`` is replaced with the zero-padded cast number. See :func:`~ctdcast.readers.ladcp.find_ladcp_file`. ship_track_nc: Path to a ship-track netCDF for the Leaflet map background line. generate: Dict of booleans controlling which page types to build. Keys: ``"stations"``, ``"sections"``, ``"timeseries"``, ``"index"``, ``"map"``. Missing keys default to ``True``. force: Regenerate all pages regardless of file modification times. skip_existing: If True, skip any page whose output HTML already exists, regardless of whether the source files are newer. Use this to fill in only missing pages without touching anything already generated. Takes precedence over the mtime check but is overridden by ``force``. section_style: ``"pcolormesh"`` or ``"contourf"`` for section figures. timeseries_style: ``"pcolormesh"`` or ``"contourf"`` for time series figures. vmin_override, vmax_override: Per-variable colormap limit overrides (e.g. ``{"SA": 34.5}``). cruise_info: Cruise metadata dict (from the ``cruise_info:`` block in ``config.yaml``). config: Frozen display configuration (GEBCO path, figsizes, map bounds, colormaps). Defaults to DEFAULT_REPORT_CONFIG. cast_filter: If set, rebuild only the station page for this cast number (implies ``generate={"stations": True, rest False}``). sal_range: ``(sal_min, sal_max)`` — records with ``ctd_salinity_1`` outside this range are excluded from all station page plots. The NC files are not modified. Excluded record count is shown in each page header. trim_soak: If True, apply pre-soak detection on each cast: cut the first 60 s (pump activation) and any records up to the last near-surface record after pump-on. Passed through to :func:`~ctdcast.reports.cast.generate_station_page`. dbar_step: Subsample the pressure axis by this step for section and timeseries plots (default 1, full 1-dbar resolution). ``build_profiles()`` always stores 1-dbar data; this controls plot-time resolution only. drop_stub: If True, a cast-page section that applies but whose figures all failed to render is dropped and the survivors renumber over it, instead of keeping the heading with an "unavailable" placeholder. Passed through to :func:`~ctdcast.reports.cast.generate_station_page`. generated_at: Page build stamp written to every page; ``None`` (the normal run) uses the current UTC time. Set to a fixed string for a reproducible build (the demo pins it so regenerating does not churn the committed HTML). Returns ------- int The number of pages that failed to build (0 on full success), so the CLI can exit non-zero when a requested page could not be generated. """ # `groupings_yaml` names what the file actually holds -- sections *and* # timeseries. `section_yaml` named half of it; accepted with a deprecation # warning until 0.5.0 (same notice as the config-key path in config.loader). if groupings_yaml is None and section_yaml is not None: warn_section_yaml_deprecated() groupings_yaml = groupings_yaml or section_yaml cfg = config if config is not None else DEFAULT_REPORT_CONFIG _figdebug.clear() # reset the per-figure debug registry for this build _failed = 0 gen: dict[str, bool] = { "stations": True, "sections": True, "timeseries": True, "index": True, "map": True, } if generate is not None: gen.update(generate) if cast_filter is not None: gen = { "stations": True, "sections": False, "timeseries": False, "index": False, "map": False, } vmin_override = vmin_override or {} vmax_override = vmax_override or {} # Resolve the build stamp once so every page in this suite carries the same # value (and a fixed stamp, for the reproducible demo, is honoured everywhere). generated_at = report_generated_at(generated_at) out_dir.mkdir(parents=True, exist_ok=True) # The "Data inventory:" strip pills (one per compiled product present) are # built once here so every page — not just the inventory pages — can render # the strip beneath its masthead. Derived from the compiled paths, so it is # available before the first page is written. # ``ctdcast fit`` writes fits.nc at the top of the CTD root (``nc_dir``), which is where it is # looked for here — not beside profiles.nc, which the config may place elsewhere. When it exists # the masthead gains a Corrections: row and corrections_ts.html is drawn from it. fits_path = Path(nc_dir) / FITS_FILENAME inventory_datasets = inventory_pills(profiles_path, ladcp_profiles_path, fits_path) # Load sections YAML once. SectionsConfig returns empty defaults when absent. _sections_cfg = ( SectionsConfig.from_yaml(groupings_yaml) if groupings_yaml else SectionsConfig() ) # cruise_info: explicit param wins; YAML cruise_info: block is the fallback. # None → not provided: use YAML block entirely. # {} → explicitly empty: suppress YAML block (caller opted out). # {...} → caller-provided: merge YAML as base, caller's keys override. if cruise_info is None: cruise_info = _sections_cfg.cruise_info elif cruise_info: cruise_info = {**_sections_cfg.cruise_info, **cruise_info} # else: cruise_info == {} → keep as-is, YAML block suppressed cast_files = _select_cast_files(nc_dir) have_per_cast = bool(cast_files) station_pages_note: str | None = None if have_per_cast: print(f"Found {len(cast_files)} cast files") all_meta_raw = [_read_cast_meta(p) for p in cast_files] all_meta = sorted( [m for m in all_meta_raw if m is not None], key=lambda m: m["time_start"], reverse=True, ) elif profiles_path is not None and Path(profiles_path).exists(): # No per-cast files: build the cruise-level (synthesis) report from the # compiled profiles.nc. Per-cast station pages read raw stage files to # reveal data warts (soak, spikes, down/up mismatch) that binning averages # away, so they are NOT built from the grid — and the index says so. print( f"No per-cast files in {nc_dir}; building cruise-level report from " f"{Path(profiles_path).name} (no station pages)." ) all_meta = sorted( _read_meta_from_profiles(Path(profiles_path)), key=lambda m: m["time_start"], reverse=True, ) station_pages_note = ( "Per-cast station pages are not shown: they require the per-cast files " f"(none were found in {nc_dir}) and are deliberately not built from the " f"compiled {Path(profiles_path).name}, because they exist to show " "raw-scan data quality that gridding averages away. The cruise-level " "pages here are compiled from the gridded profiles." ) else: # `report` only draws pages; say what would have made the files it looked for. where = Path(nc_dir) state = ( "the folder does not exist" if not where.is_dir() else "it holds no stage1/, stage2/ or stage3/ cast files and no profiles.nc" ) print( f"No processed casts to report from {where}: {state}.\n" " `ctdcast report` draws pages from processed files; it does not read raw " "CNV or .hex.\n" " Process first with `ctdcast run <config>` (process, then report), or " "`ctdcast process <config> --stage 1 2 3 profiles`." ) return None if not all_meta: print(f"No casts found for {nc_dir}") return None _nc_cruise = all_meta[0].get("cruise") if all_meta else None cruise = cruise_name(cruise_info) or _nc_cruise or UNKNOWN_CRUISE_ID # Pre-load GEBCO for the cruise area into memory so every map figure # subsets from numpy arrays rather than reopening the file from disk. from ctdcast.analysis.bathymetry import preload_gebco _gebco_path = cfg.gebco_path if _gebco_path is not None and all_meta: _cast_lats = [ m["lat"] for m in all_meta if np.isfinite(m.get("lat", float("nan"))) ] _cast_lons = [ m["lon"] for m in all_meta if np.isfinite(m.get("lon", float("nan"))) ] if _cast_lats: # The cruise maps frame to a square (in Mercator proportion) around # the data plus up to a ~30% margin. Preload that framed extent, not # just the raw data box, so the bathymetry fills each map instead of # leaving a grey border — most visible for a tall cruise, where the # square widens longitude well past the data. from ctdcast.plotters.maps import ( GEBCO_RENDER_PAD, square_map_limits, ) _la0, _la1 = float(min(_cast_lats)), float(max(_cast_lats)) _lo0, _lo1 = float(min(_cast_lons)), float(max(_cast_lons)) _mrg = max(0.5, max(_la1 - _la0, _lo1 - _lo0) * 0.30) _cos = float(max(0.15, np.cos(np.radians((_la0 + _la1) / 2.0)))) _py0, _py1, _px0, _px1 = square_map_limits( _la0 - _mrg, _la1 + _mrg, _lo0 - _mrg, _lo1 + _mrg, _cos ) _t0 = perf_counter() preload_gebco(_gebco_path, _py0, _py1, _px0, _px1, margin=GEBCO_RENDER_PAD) print(f" GEBCO preloaded ({perf_counter() - _t0:.1f}s)") # Build cruise-wide cast_notes mapping from all sections and timeseries. all_cast_notes: dict[int, list[str]] = {} for _grp_dict in (_sections_cfg.sections, _sections_cfg.timeseries): for _cfg in _grp_dict.values(): if not isinstance(_cfg, dict): continue for _cn, _note in (_cfg.get("cast_notes") or {}).items(): _cn_int = int(_cn) all_cast_notes.setdefault(_cn_int, []) if _note and _note not in all_cast_notes[_cn_int]: all_cast_notes[_cn_int].append(str(_note)) if gen["stations"] and have_per_cast: _t0 = perf_counter() cast_num_strs = [m["cast_num_str"] for m in all_meta] _cast_set: set[int] | None = ( {cast_filter} if isinstance(cast_filter, int) else set(cast_filter) if cast_filter is not None else None ) targets = [ m for m in all_meta if _cast_set is None or m["cast_num"] in _cast_set ] if _cast_set is not None and not targets: print(f"Cast(s) {cast_filter} not found in {nc_dir}") return None for meta in targets: orig_i = all_meta.index(meta) prev_cast_str = cast_num_strs[orig_i - 1] if orig_i > 0 else None next_cast_str = ( cast_num_strs[orig_i + 1] if orig_i < len(all_meta) - 1 else None ) _expected = out_dir / "casts" / f"cast_{meta['cast_num_str']}.html" _was_new = not _expected.exists() _html_mtime_str = _fmt_mtime( _expected ) # capture before page is (re)written # Resolve LADCP mat path before the skip check so it feeds the mtime comparison. _mat: Path | None = None if ladcp_dir is not None: _mat = find_ladcp_file( ladcp_dir, meta["cast_num"], meta["cast_suffix"], ladcp_pattern ) _extra: tuple[Path, ...] = (_mat,) if _mat is not None else () _skip_reason = _mtime_skip_reason( _expected, meta["path"], force, *_extra, skip_existing=skip_existing ) _ladcp_note = "" if ( _skip_reason and _mat is not None and _expected.exists() and b"s-ladcp" not in _expected.read_bytes() ): _ladcp_note = " — LADCP available, use --force" out_page = generate_station_page( meta["path"], out_dir, all_meta, prev_cast_str=prev_cast_str, next_cast_str=next_cast_str, force=force or not _skip_reason, ladcp_dir=ladcp_dir, ladcp_pattern=ladcp_pattern, cast_num_str=meta["cast_num_str"], sal_range=sal_range, trim_soak=trim_soak, cast_notes=all_cast_notes.get(meta["cast_num"]), cruise_info=cruise_info, drop_stub=drop_stub, inventory=inventory_datasets, generated_at=generated_at, cfg=cfg, ) if _skip_reason: _status = _skip_reason + _ladcp_note elif out_page is None: _status = "FAILED" _failed += 1 elif force: _status = "regenerated (forced)" elif _was_new: _ladcp_part = f", ladcp: {_fmt_mtime(_mat)}" if _mat else "" _status = ( f"regenerated (new) [nc: {_fmt_mtime(meta['path'])}{_ladcp_part}]" ) else: _ladcp_part = f", ladcp: {_fmt_mtime(_mat)}" if _mat else "" _status = ( f"regenerated (source updated)" f" [html: {_html_mtime_str}," f" nc: {_fmt_mtime(meta['path'])}{_ladcp_part}]" ) print(f" cast {meta['cast_num_str']}: {_status}") print(f" [casts: {perf_counter() - _t0:.1f}s]") sections_cfg: dict[str, Any] = _sections_cfg.sections timeseries_cfg: dict[str, Any] = _sections_cfg.timeseries if gen["sections"]: _t0 = perf_counter() _sec_names = list(sections_cfg.keys()) for _sec_i, (sec_name, sec_cfg) in enumerate(sections_cfg.items()): _sec_prev = _sec_names[_sec_i - 1] if _sec_i > 0 else None _sec_next = _sec_names[_sec_i + 1] if _sec_i < len(_sec_names) - 1 else None _expected = out_dir / "sections" / f"section_{sec_name}.html" _sec_was_new = not _expected.exists() _sec_html_mtime_str = _fmt_mtime(_expected) _src = profiles_path if profiles_path is not None else _expected _sec_skip = _mtime_skip_reason( _expected, _src, force, skip_existing=skip_existing ) _sec_note = "" if _sec_skip and ladcp_dir is not None: _sec_casts = expand_cast_numbers(sec_cfg.get("cast_numbers", [])) if ( any( find_ladcp_file(ladcp_dir, cn, ladcp_pattern=ladcp_pattern) is not None for cn in _sec_casts ) and _expected.exists() and b"s-ladcp" not in _expected.read_bytes() ): _sec_note = " — LADCP available, use --force" out_page = generate_section_page( sec_name, sec_cfg, profiles_path, out_dir, force=force or not _sec_skip, section_style=section_style, vmin_override=vmin_override, vmax_override=vmax_override, ladcp_dir=ladcp_dir, ladcp_pattern=ladcp_pattern, dbar_step=dbar_step, prev_name=_sec_prev, next_name=_sec_next, cruise_info=cruise_info, inventory=inventory_datasets, generated_at=generated_at, cfg=cfg, ) if _sec_skip: _sec_status = _sec_skip + _sec_note elif out_page is None: _sec_status = "FAILED" _failed += 1 elif force: _sec_status = "regenerated (forced)" elif _sec_was_new: _sec_status = f"regenerated (new) [src: {_fmt_mtime(_src)}]" else: _sec_status = ( f"regenerated (source updated)" f" [html: {_sec_html_mtime_str}, src: {_fmt_mtime(_src)}]" ) print(f" section {sec_name}: {_sec_status}") print(f" [sections: {perf_counter() - _t0:.1f}s]") if gen["timeseries"] and timeseries_cfg: _t0 = perf_counter() _ts_names = list(timeseries_cfg.keys()) for _ts_i, (ts_name, ts_cfg) in enumerate(timeseries_cfg.items()): _ts_prev = _ts_names[_ts_i - 1] if _ts_i > 0 else None _ts_next = _ts_names[_ts_i + 1] if _ts_i < len(_ts_names) - 1 else None _expected = out_dir / "timeseries" / f"timeseries_{ts_name}.html" _ts_was_new = not _expected.exists() _ts_html_mtime_str = _fmt_mtime(_expected) _src = profiles_path if profiles_path is not None else _expected _ts_skip = _mtime_skip_reason( _expected, _src, force, skip_existing=skip_existing ) _ts_note = "" if _ts_skip and ladcp_dir is not None: _ts_casts = expand_cast_numbers(ts_cfg.get("cast_numbers", [])) if ( any( find_ladcp_file(ladcp_dir, cn, ladcp_pattern=ladcp_pattern) is not None for cn in _ts_casts ) and _expected.exists() and b"s-ladcp" not in _expected.read_bytes() ): _ts_note = " — LADCP available, use --force" out_page = generate_timeseries_page( ts_name, ts_cfg, profiles_path, out_dir, force=force or not _ts_skip, section_style=timeseries_style, vmin_override=vmin_override, vmax_override=vmax_override, all_meta=all_meta, ladcp_dir=ladcp_dir, ladcp_pattern=ladcp_pattern, dbar_step=dbar_step, prev_name=_ts_prev, next_name=_ts_next, cruise_info=cruise_info, inventory=inventory_datasets, generated_at=generated_at, cfg=cfg, ) if _ts_skip: _ts_status = _ts_skip + _ts_note elif out_page is None: _ts_status = "FAILED" _failed += 1 elif force: _ts_status = "regenerated (forced)" elif _ts_was_new: _ts_status = f"regenerated (new) [src: {_fmt_mtime(_src)}]" else: _ts_status = ( f"regenerated (source updated)" f" [html: {_ts_html_mtime_str}, src: {_fmt_mtime(_src)}]" ) print(f" timeseries {ts_name}: {_ts_status}") print(f" [timeseries: {perf_counter() - _t0:.1f}s]") if gen["index"]: # Generate a netCDF inventory page per compiled dataset present. Both the # strip pills (``inventory_datasets``, built at the top of report()) and this # generation loop read the same existence rule via inventory_sources(), so a # strip pill can never point at a page that was not generated, or vice versa. _dataset_srcs, _has_sensors = inventory_sources( profiles_path, ladcp_profiles_path ) for _src in _dataset_srcs: _href = f"{_src.stem}_inventory.html" _t0 = perf_counter() try: from ctdcast.reports.dataset import generate_dataset_page generate_dataset_page( _src, out_dir / _href, title=f"{_src.name} — {cruise} data inventory", cruise=cruise, nav_prefix="", show_nav=True, inventory_pills=inventory_datasets, current_href=_href, generated_at=generated_at, # The profiles inventory points at the sensors page for its sensor # variables rather than repeating the tables there. sensors_href=( "sbe_sensors.html" if _has_sensors and Path(_src) == Path(profiles_path) else None ), ) print(f" [{_href}: {perf_counter() - _t0:.1f}s]") except Exception: # noqa: BLE001 import traceback print(f" {_href}: FAILED") _failed += 1 traceback.print_exc() # SBE sensors page — reads the sensor catalog/linkage from profiles.nc and # renders the configuration / inventory / rewiring tables. if _has_sensors: _t0 = perf_counter() try: from ctdcast.reports._sensors import generate_sensors_page generate_sensors_page( Path(profiles_path), out_dir / "sbe_sensors.html", cruise=cruise, nav_prefix="", show_nav=True, inventory_pills=inventory_datasets, current_href="sbe_sensors.html", generated_at=generated_at, inventory_href=f"{Path(profiles_path).stem}_inventory.html", ) print(f" [sbe_sensors.html: {perf_counter() - _t0:.1f}s]") except Exception: # noqa: BLE001 import traceback print(" sbe_sensors.html: FAILED") _failed += 1 traceback.print_exc() # Processing page — every step applied, by run of casts, from the # processing_record the profiles compile stores per cast. _t0 = perf_counter() try: from ctdcast.reports.processing import ( PROCESSING_HREF, generate_processing_page, ) generate_processing_page( Path(profiles_path), out_dir / PROCESSING_HREF, cruise=cruise, nav_prefix="", show_nav=True, inventory_pills=inventory_datasets, current_href=PROCESSING_HREF, generated_at=generated_at, ) print(f" [{PROCESSING_HREF}: {perf_counter() - _t0:.1f}s]") except Exception: # noqa: BLE001 import traceback print(" processing.html: FAILED") _failed += 1 traceback.print_exc() if fits_path is not None and fits_path.exists(): _t0 = perf_counter() try: from ctdcast.reports.corrections import generate_corrections_page generate_corrections_page( fits_path, out_dir / CORRECTIONS_TS_HREF, cruise=cruise, inventory_pills=inventory_datasets, generated_at=generated_at, ) print(f" [{CORRECTIONS_TS_HREF}: {perf_counter() - _t0:.1f}s]") except Exception: # noqa: BLE001 import traceback print(f" {CORRECTIONS_TS_HREF}: FAILED") _failed += 1 traceback.print_exc() _t0 = perf_counter() _write_index( all_meta, sections_cfg, cruise, out_dir, force, profiles_path=profiles_path, ladcp_profiles_path=ladcp_profiles_path, section_style=section_style, vmin_override=vmin_override, vmax_override=vmax_override, cruise_info=cruise_info, timeseries_cfg=timeseries_cfg, inventory_datasets=inventory_datasets, station_pages_note=station_pages_note, generated_at=generated_at, cfg=cfg, ) print(f" [index.html: {perf_counter() - _t0:.1f}s]") _t0 = perf_counter() _write_stations_list( all_meta, cruise, out_dir, sections_cfg=sections_cfg, timeseries_cfg=timeseries_cfg, ladcp_dir=ladcp_dir, ladcp_pattern=ladcp_pattern, cruise_info=cruise_info, no_station_pages=not have_per_cast, nc_dir=nc_dir, inventory_datasets=inventory_datasets, generated_at=generated_at, cfg=cfg, ) print(f" [casts.html: {perf_counter() - _t0:.1f}s]") _t0 = perf_counter() _write_sections_list( sections_cfg, cruise, out_dir, all_meta=all_meta, ladcp_dir=ladcp_dir, cruise_info=cruise_info, inventory_datasets=inventory_datasets, generated_at=generated_at, cfg=cfg, ) print(f" [sections.html: {perf_counter() - _t0:.1f}s]") _t0 = perf_counter() _write_timeseries_list( timeseries_cfg, cruise, out_dir, all_meta=all_meta, ladcp_dir=ladcp_dir, cruise_info=cruise_info, inventory_datasets=inventory_datasets, generated_at=generated_at, cfg=cfg, ) print(f" [timeseries.html: {perf_counter() - _t0:.1f}s]") if gen["map"]: try: from ctdcast.reports.cruise_map import generate_leaflet_map lf_out = generate_leaflet_map( all_meta, sections_cfg, out_dir, force=force, ship_track_nc=ship_track_nc, cruise=cruise, inventory=inventory_datasets, generated_at=generated_at, cfg=cfg, ) print(f" leaflet map: {'regenerated' if lf_out else 'skipped (no casts)'}") except Exception: # noqa: BLE001 import traceback print(" leaflet map: FAILED") _failed += 1 traceback.print_exc() print(f"\nReport written to {out_dir}/index.html") return _failed
# --------------------------------------------------------------------------- # Page writers # --------------------------------------------------------------------------- def _write_index( all_meta: list[dict[str, Any]], sections_cfg: dict[str, Any], cruise: str, out_dir: Path, force: bool, # noqa: ARG001 profiles_path: Path | None = None, ladcp_profiles_path: Path | None = None, section_style: str = "pcolormesh", vmin_override: dict[str, float] | None = None, vmax_override: dict[str, float] | None = None, cruise_info: dict[str, Any] | None = None, timeseries_cfg: dict[str, Any] | None = None, inventory_datasets: list[dict[str, str]] | None = None, station_pages_note: str | None = None, generated_at: str | None = None, cfg: ReportConfig = DEFAULT_REPORT_CONFIG, ) -> None: """Write index.html with header card, stats, overview map, and stacked property panels. *inventory_datasets* is a list of ``{"label", "href"}`` dicts, one per compiled netCDF product, rendered as a strip of data-inventory pills on the summary page. """ times = [m["time_start"] for m in all_meta if m.get("time_start")] times_str = sorted(str(t)[:10] for t in times if t) n_days = 0 if len(times_str) >= 2: from datetime import date try: d0 = date.fromisoformat(times_str[0]) # earliest d1 = date.fromisoformat(times_str[-1]) # latest n_days = (d1 - d0).days + 1 except ValueError: pass max_depth = max((m.get("max_depth", 0) for m in all_meta), default=0) ci = cruise_info or {} _valid_lats = [m["lat"] for m in all_meta if np.isfinite(m.get("lat", np.nan))] _valid_lons = [m["lon"] for m in all_meta if np.isfinite(m.get("lon", np.nan))] # Build combined sections+timeseries data for the map and for overview markers _ts_cfg = timeseries_cfg or {} _all_groups = {**sections_cfg, **_ts_cfg} sections_data_map: list[dict[str, Any]] = [] # cast_groups: color → set of cast numbers (for overview panel top markers) cast_groups: dict[str, list[int]] = {} for grp_cfg in _all_groups.values(): cast_nums_grp = expand_cast_numbers(grp_cfg.get("cast_numbers", [])) color = grp_cfg.get("color", "#888888") if cast_nums_grp: cast_groups[color] = cast_groups.get(color, []) + list(cast_nums_grp) # For the index map, gather section track positions from all_meta cast_pos = { m["cast_num"]: (m["lat"], m["lon"]) for m in all_meta if np.isfinite(m.get("lat", np.nan)) } for grp_name, grp_cfg in sections_cfg.items(): cast_nums_grp = expand_cast_numbers(grp_cfg.get("cast_numbers", [])) lats_g = [cast_pos[cn][0] for cn in sorted(cast_nums_grp) if cn in cast_pos] lons_g = [cast_pos[cn][1] for cn in sorted(cast_nums_grp) if cn in cast_pos] if lats_g: sections_data_map.append( { "name": grp_name, "color": grp_cfg.get("color", "#888888"), "lats": lats_g, "lons": lons_g, } ) for grp_name, grp_cfg in _ts_cfg.items(): cast_nums_grp = expand_cast_numbers(grp_cfg.get("cast_numbers", [])) lats_g = [cast_pos[cn][0] for cn in sorted(cast_nums_grp) if cn in cast_pos] lons_g = [cast_pos[cn][1] for cn in sorted(cast_nums_grp) if cn in cast_pos] if lats_g: sections_data_map.append( { "name": grp_name, "color": grp_cfg.get("color", "#888888"), "lats": lats_g, "lons": lons_g, } ) if sections_data_map and _valid_lats: fig_map_b64 = _make_all_sections_map_b64( sections_data_map, _valid_lats, _valid_lons, legend_outside=True, target_h=3.0, cfg=cfg, width_in=_slot_width(_MAP_SLOT), ) elif _valid_lats: # No sections configured (e.g. draft mode) — fall back to cruise-track map. fig_map_b64 = _make_cruise_map_b64( all_meta, target_h=3.0, cfg=cfg, width_in=_slot_width(_MAP_SLOT) ) else: fig_map_b64 = None # Stacked overview panels and cruise T-S diagram from profiles.nc. All are # optional (a None b64 is dropped, never stubbed), so they are rendered here and # the non-None ones fed to the manifest as pre-rendered PanelGroups. physics_panels_idx: tuple[RenderedPanel, ...] = () biogeo_panels_idx: tuple[RenderedPanel, ...] = () ts_panels_idx: tuple[RenderedPanel, ...] = () if profiles_path is not None and profiles_path.exists(): try: ds_all = xr.open_dataset( profiles_path, decode_timedelta=False, engine="netcdf4" ).load() ds_all = add_teos10_profiles(fill_oxygen_composite(ds_all)) ds_all = add_aou(ds_all) mask_down = ds_all["cast_type"].values == "down" ds_down = ds_all.isel(N_PROF=mask_down) order = np.argsort(ds_down["cast_number"].values) ds_sorted = ds_down.isel(N_PROF=order) lats = ds_sorted["latitude"].values.tolist() lons = ds_sorted["longitude"].values.tolist() bathy = interpolate_bathy_at_casts(lats, lons, path=cfg.gebco_path) vmin = vmin_override or {} vmax = vmax_override or {} def _idx_panel(var: str) -> RenderedPanel: # Resolve the single/dual-sensor form actually present (e.g. a # single-oxygen cruise stores plain ``ctd_oxygen``, not # ``ctd_oxygen_1``), the way the section page does — otherwise the # overview silently drops a variable it holds under the other form. resolved = resolve_sensor_var(ds_sorted, var) # An override may be keyed by either sensor form; same lookup as the # section page. _vmin = color_limit_override(vmin, var, resolved) _vmax = color_limit_override(vmax, var, resolved) b64 = _make_overview_panel_b64( ds_sorted, resolved, vlabel(resolved), bathy_depths=bathy, style=section_style, vmin=_vmin, vmax=_vmax, cast_groups=cast_groups, cfg=cfg, width_in=_slot_width(_OVERVIEW_SLOT), ) # title is the panel's HTML caption on index.html, so use the # HTML-ready label (mathtext subscripts rendered as Unicode). return RenderedPanel( b64=b64, title=vlabel_html(resolved), short=resolved, slot=_OVERVIEW_SLOT, ) physics_panels_idx = tuple( p for p in (_idx_panel(v) for v in SECTION_PHYSICS_VARS) if p.b64 ) biogeo_panels_idx = tuple( p for p in (_idx_panel(v) for v in SECTION_BIOGEO_VARS) if p.b64 ) _ts_b64 = _make_section_ts_histogram_b64( ds_sorted, cfg=cfg, width_in=_slot_width(_TS_SLOT) ) ts_panels_idx = ( ( RenderedPanel( b64=_ts_b64, title="All downcast profiles", slot=_TS_SLOT ), ) if _ts_b64 else () ) ds_all.close() except Exception: # noqa: BLE001 pass # Overview panels are optional; never crash index generation # Cruise-wide LADCP velocity overview (compiled ladcp_profiles.nc). Optional # and rendered up front like the other overview panels — an empty list drops # the Velocity section rather than stubbing it. velocity_panels_idx: tuple[RenderedPanel, ...] = () if ladcp_profiles_path is not None and Path(ladcp_profiles_path).exists(): try: ds_ladcp = xr.open_dataset( ladcp_profiles_path, decode_timedelta=False, engine="netcdf4" ).load() velocity_panels_idx = tuple( dataclasses.replace(p, slot=_OVERVIEW_SLOT) for p in _make_ladcp_overview_b64( ds_ladcp, cast_groups=cast_groups, style=section_style, cfg=cfg, width_in=_slot_width(_OVERVIEW_SLOT), ) ) ds_ladcp.close() except Exception: # noqa: BLE001 pass # Velocity panels are optional; never crash index generation date_start = times_str[0] if times_str else "" date_end = times_str[-1] if len(times_str) >= 2 else "" page_ctx = IndexPageCtx( map_b64=fig_map_b64, physics_panels=physics_panels_idx, biogeo_panels=biogeo_panels_idx, velocity_panels=velocity_panels_idx, ts_panels=ts_panels_idx, ) report = resolve_index(page_ctx) ctx: dict[str, Any] = { "cruise": cruise, "ship": ci.get("ship", ""), "project": ci.get("project", ""), "date_start": date_start, "date_end": date_end, "n_casts": len(all_meta), "n_sections": len(sections_cfg), "max_depth_str": f"{max_depth:.0f} dbar", "n_days": n_days, "report": report, "inventory_datasets": inventory_datasets or [], "station_pages_note": station_pages_note, "version": _VERSION, "generated_at": report_generated_at(generated_at), } html = get_template("index.html").render( **ctx, css=SHARED_CSS, js_top_links=_JS_TOP_LINKS, nav_prefix="", nav=nav_for("summary", prefix="", inventory=inventory_datasets or ()), ) (out_dir / "index.html").write_text(html, encoding="utf-8") # --------------------------------------------------------------------------- # Index manifest — the cruise summary page as data (mirrors _section.py) # --------------------------------------------------------------------------- @dataclasses.dataclass(frozen=True) class IndexPageCtx: """Per-cruise render context for index.html: the pre-rendered overview panels. Every panel is optional and rendered up front, so the context carries finished base64 payloads (not a dataset); the manifest only numbers, anchors and lays them out. """ map_b64: str | None physics_panels: tuple[RenderedPanel, ...] biogeo_panels: tuple[RenderedPanel, ...] velocity_panels: tuple[RenderedPanel, ...] ts_panels: tuple[RenderedPanel, ...] def _index_overview_panel(rp: RenderedPanel) -> Panel: """Wrap a pre-rendered cruise overview panel, captioned by variable, at the slot it was drawn for.""" return Panel( id=f"index_{rp.short or rp.title}", slot=rp.slot, caption=rp.title, render=lambda _c, _b64=rp.b64: _b64, ) def _index_ts_panel(rp: RenderedPanel) -> Panel: """Wrap the pre-rendered cruise T–S histogram in a panel at the slot it was drawn for.""" return Panel( id="index_ts", slot=rp.slot, caption=rp.title, render=lambda _c, _b64=rp.b64: _b64, ) #: Only the map is fixed; the overview fields and T–S are data-driven groups. INDEX_PANELS: dict[str, Panel] = { "index_map": Panel( id="index_map", slot=_MAP_SLOT, applies_to=lambda c: c.map_b64 is not None, render=lambda c: c.map_b64, ), } #: The cruise summary profile. Same section ids as the section/timeseries pages, #: so the shared anchors and legacy #s-* aliases resolve consistently. INDEX_DEFAULT: Profile = Profile( numbering="flat", entries=( Section( "map", "Map", ("index_map",), intro="Cruise track and station positions.", ), Section( "hydrography", "Hydrography", (PanelGroup(over=lambda c: c.physics_panels, panel=_index_overview_panel),), intro=( "Cruise-wide overview of conservative temperature, absolute salinity " "and potential density across all downcast stations." ), ), Section( "biogeochemistry", "Biogeochemistry", (PanelGroup(over=lambda c: c.biogeo_panels, panel=_index_overview_panel),), intro=( "Cruise-wide overview of oxygen, fluorescence and turbidity across all " "downcast stations." ), ), Section( "velocity", "Velocity", ( PanelGroup( over=lambda c: c.velocity_panels, panel=_index_overview_panel ), ), intro=( "Cruise-wide eastward (U) and northward (V) velocity from the " "LADCP inverse solution across all stations." ), ), Section( "ts_diagram", "T–S diagram", (PanelGroup(over=lambda c: c.ts_panels, panel=_index_ts_panel),), intro="CT–SA histogram of all downcast profiles.", ), ), ) def resolve_index(ctx: IndexPageCtx) -> ResolvedReport: """Resolve the index profile against *ctx* into numbered, rendered sections.""" return resolve(INDEX_DEFAULT, ctx, INDEX_PANELS) def _dec_min(deg: float, pos_hem: str, neg_hem: str) -> str: """Format decimal degrees as degrees-decimal-minutes, e.g. 65°13.782′N.""" if not np.isfinite(deg): return "—" hem = pos_hem if deg >= 0 else neg_hem deg = abs(deg) d = int(deg) m = (deg - d) * 60.0 return f"{d}°{m:06.3f}′{hem}" _DEPTH_PILL_PALETTE: list[tuple[str, str]] = [ ("#deebf7", "#1a1a2e"), # shallowest — very light blue, dark text ("#9ecae1", "#1a1a2e"), # light blue, dark text ("#4292c6", "#ffffff"), # medium blue, white text ("#2171b5", "#ffffff"), # dark blue, white text ("#084594", "#ffffff"), # deepest — very dark blue, white text ] def _depth_pill_style(depth: float, rounded_max: float) -> tuple[str, str]: """Return (background_hex, text_hex) for a depth pill based on 5-class scheme.""" if rounded_max <= 0 or not np.isfinite(depth): return _DEPTH_PILL_PALETTE[0] cls = min(4, int(depth / rounded_max * 5)) return _DEPTH_PILL_PALETTE[cls] def _round_max_depth(max_depth: float) -> float: """Round *max_depth* up to the nearest 100, 500, or 1000 dbar.""" if max_depth <= 500: return float(np.ceil(max_depth / 100) * 100) if max_depth <= 2000: return float(np.ceil(max_depth / 500) * 500) return float(np.ceil(max_depth / 1000) * 1000) def _make_clock_section( nc_dir: Path, cfg: ReportConfig = DEFAULT_REPORT_CONFIG ) -> dict[str, Any] | None: """Build the acquisition-clock section for casts.html, or ``None`` when there is no stage-1 tree. Reads each stage-1 header's System/NMEA clock pair, classifies the cruise, and returns the caption facts, the segment and per-cast tables, the offset figure, and the paste-ready config block. Returns ``None`` when no stage-1 files exist (e.g. a profiles-only report), so the section is simply absent rather than reporting "not checked". """ from ctdcast.analysis.clock import ( classify_offsets, clock_offsets, coordinate_summary, correction_status, suggested_config_yaml, ) from ctdcast.config.cnv_header import header_from_raw_metadata, parse_start_time from ctdcast.plotters.timeseries import draw_clock_offset_fig from ctdcast.processors.stage_layout import group_by_cast from ctdcast.reports._vendored.figure_png import render_b64 # Discover casts the same way clock_offsets does (nested, flat, or stage-2/3-only trees); absent # only when there are no casts at all, in which case the section is simply not shown. groups = group_by_cast(nc_dir) if not groups: return None series, scanned, coordinate_counts = clock_offsets(nc_dir) verdict = classify_offsets(series, n_scanned=scanned) # Caption facts: cruise-constant. Read from the first cast's start_time bracket (present on # every file via raw_metadata) rather than a global attr, so this works on trees predating the # attr. The bracket names which clock the coordinate is anchored to — the fact that decides # whether the offsets below are a defect or a curiosity. first_stages = groups[min(groups)] caption_path = first_stages.get(1, first_stages[min(first_stages)]) caption: dict[str, Any] = {} ds = xr.open_dataset(caption_path, engine="netcdf4") try: start = parse_start_time( header_from_raw_metadata(ds.attrs.get("raw_metadata")) or "" ) if start.source: caption["start_line"] = f"{start.value} [{start.source}]" source_var = ds.attrs.get("cnv_time_source_variable") if source_var: caption["source_variable"] = str(source_var) finally: ds.close() segments = [ { "cast_range": s.cast_range, "first_utc": f"{s.start:%Y-%m-%d %H:%M}", "offset": f"{s.offset_seconds:+.2f}", "sd": f"{s.sd_seconds:.2f}", "n": s.n_casts, } for s in verdict.segments ] casts = [ { "cast_id": c.cast_id, "start": f"{c.system_utc:%Y-%m-%d}", "system": f"{c.system_utc:%H:%M:%S}", "nmea": f"{c.nmea_utc:%H:%M:%S}", "offset": f"{c.offset_seconds:+.0f}", } for c in series ] suggested = suggested_config_yaml(verdict) # A coordinate already on GPS (or of undetermined source) gets no paste-ready block, so the # section never contradicts the coordinate line, nor asserts a source that was not measured. # The one gate both the CLI and this page read, so they cannot disagree. status = correction_status(coordinate_counts) figure_b64 = ( render_b64( draw_clock_offset_fig, series, verdict, cfg=cfg, width_in=_slot_width(_CLOCK_SLOT), optional=True, ) if verdict.segments else None ) return { "kind": verdict.kind, "note": verdict.note, "caveat": verdict.resolution_caveat, "scanned": scanned, "n_pairs": len(series), "coordinate_line": coordinate_summary(coordinate_counts, scanned), "caption": caption, "segments": segments, "casts": casts, "figure_b64": figure_b64, "slot": _CLOCK_SLOT, "suggested_yaml": suggested if status == "apply" else None, "correction_status": status, } def _write_stations_list( all_meta: list[dict[str, Any]], cruise: str, out_dir: Path, sections_cfg: dict[str, Any] | None = None, timeseries_cfg: dict[str, Any] | None = None, ladcp_dir: Path | None = None, ladcp_pattern: str | None = None, cruise_info: dict[str, Any] | None = None, no_station_pages: bool = False, nc_dir: Path | None = None, inventory_datasets: list[dict[str, str]] | None = None, generated_at: str | None = None, cfg: ReportConfig = DEFAULT_REPORT_CONFIG, ) -> None: """Write casts.html with cruise map, depth pills, and section/timeseries links. When *no_station_pages* is true (a report compiled from ``profiles.nc`` with no per-cast files), the cast rows drop their link to the per-cast page — those pages are not produced — and a note explains why, so no row points at a file that does not exist. """ # LADCP: collect cast numbers that have a processed .mat file. # Use find_ladcp_file per cast so non-NNN.mat filenames (e.g. msm_142_1_NNN.mat) # are handled correctly. ladcp_cast_nums: set[int] = set() if ladcp_dir is not None and ladcp_dir.exists(): for meta in all_meta: cn = meta["cast_num"] cs = meta.get("cast_suffix", "") if find_ladcp_file(ladcp_dir, cn, cs, ladcp_pattern) is not None: ladcp_cast_nums.add(cn) # Section name → YAML color; cast → section names section_colors: dict[str, str] = { name: cfg.get("color", "#2c6e49") for name, cfg in (sections_cfg or {}).items() } cast_to_sections: dict[int, list[str]] = {} for sec_name, sec_cfg in (sections_cfg or {}).items(): for cn in expand_cast_numbers(sec_cfg.get("cast_numbers", [])): cast_to_sections.setdefault(cn, []).append(sec_name) # Timeseries name → YAML color; cast → timeseries names ts_colors: dict[str, str] = { name: cfg.get("color", "#7b2d8b") for name, cfg in (timeseries_cfg or {}).items() } cast_to_timeseries: dict[int, list[str]] = {} for ts_name, ts_cfg in (timeseries_cfg or {}).items(): for cn in expand_cast_numbers(ts_cfg.get("cast_numbers", [])): cast_to_timeseries.setdefault(cn, []).append(ts_name) all_depths = [ m.get("max_depth", 0) for m in all_meta if np.isfinite(m.get("max_depth", 0)) ] rounded_max = _round_max_depth(max(all_depths, default=100)) stations = [] for m in all_meta: lat = m.get("lat", np.nan) lon = m.get("lon", np.nan) depth = m.get("max_depth", 0) bg, fg = _depth_pill_style(float(depth), rounded_max) cast_num_int = m["cast_num"] stations.append( { "cast_num": m["cast_num_str"], "filename": m.get("raw_filename", "—"), "time_start_str": str(m.get("time_start", ""))[:16].replace("T", " "), "lat_str": _dec_min(lat, "N", "S"), "lon_str": _dec_min(lon, "E", "W"), "max_depth_str": f"{depth:.0f}", "depth_bg": bg, "depth_fg": fg, "sections": [ {"name": sn, "color": section_colors.get(sn, "#2c6e49")} for sn in cast_to_sections.get(cast_num_int, []) ], "timeseries": [ {"name": tn, "color": ts_colors.get(tn, "#7b2d8b")} for tn in cast_to_timeseries.get(cast_num_int, []) ], "ladcp_has": cast_num_int in ladcp_cast_nums, "qc_pct": _cast_qc_pct(m.get("path")), } ) _ci = cruise_info or {} _times_str = sorted( str(m["time_start"])[:10] for m in all_meta if m.get("time_start") ) _date_start = _times_str[0] if _times_str else "" _date_end = _times_str[-1] if len(_times_str) >= 2 else "" _duration_days = 0 if _date_start and _date_end: try: from datetime import date as _date _duration_days = ( _date.fromisoformat(_date_end) - _date.fromisoformat(_date_start) ).days + 1 except ValueError: pass _max_depth = max((m.get("max_depth", 0) for m in all_meta), default=0) # Interactive cast map (vendored track_map macro): one point per cast, one polyline per # section, rows hover-linked by data-track-id; the static cruise_map PNG is the print/no-JS # fallback. Lazy import mirrors generate_leaflet_map below. from ctdcast.reports._vendored.leaflet import load_leaflet from ctdcast.reports.cruise_map import CASTS_MAP_HEIGHT, casts_map_context _leaflet_js, _leaflet_css = load_leaflet() _map_ctx = casts_map_context(all_meta, sections_cfg or {}) ctx: dict[str, Any] = { "cruise": cruise, "ship": _ci.get("ship", ""), "date_start": _date_start, "date_end": _date_end, "duration_days": _duration_days, "max_depth_str": f"{_max_depth:.0f} dbar" if _max_depth else "", "stations": stations, "no_station_pages": no_station_pages, "cruise_map_b64": _make_cruise_map_b64(all_meta, target_h=3.0, cfg=cfg), "leaflet_js": _leaflet_js, "leaflet_css": _leaflet_css, "map_height": CASTS_MAP_HEIGHT, **_map_ctx, "clock": _make_clock_section(nc_dir, cfg=cfg) if nc_dir is not None else None, "ladcp_configured": ladcp_dir is not None, "version": _VERSION, "generated_at": report_generated_at(generated_at), } html = get_template("casts.html").render( **ctx, css=SHARED_CSS, js_top_links=_JS_TOP_LINKS, nav_prefix="", nav=nav_for("casts", prefix="", inventory=inventory_datasets or ()), ) (out_dir / "casts.html").write_text(html, encoding="utf-8") def _write_sections_list( sections_cfg: dict[str, Any], cruise: str, out_dir: Path, all_meta: list[dict[str, Any]] | None = None, ladcp_dir: Path | None = None, cruise_info: dict[str, Any] | None = None, inventory_datasets: list[dict[str, str]] | None = None, generated_at: str | None = None, cfg: ReportConfig = DEFAULT_REPORT_CONFIG, ) -> None: """Write sections.html with overview map and cards for each section.""" ladcp_cast_nums: set[int] = set() if ladcp_dir is not None and ladcp_dir.exists(): for f in ladcp_dir.glob("*.mat"): with contextlib.suppress(ValueError): ladcp_cast_nums.add(int(f.stem)) # Build cast → position lookup for the overview map cast_pos: dict[int, tuple[float, float]] = {} if all_meta: for m in all_meta: cn = m.get("cast_num") if ( cn is not None and np.isfinite(m.get("lat", np.nan)) and np.isfinite(m.get("lon", np.nan)) ): cast_pos[int(cn)] = (float(m["lat"]), float(m["lon"])) sections = [] sections_data: list[dict[str, Any]] = [] _sec_n_casts_list: list[int] = [] for name, grp_cfg in sections_cfg.items(): cast_nums = expand_cast_numbers(grp_cfg.get("cast_numbers", [])) report_path = out_dir / "sections" / f"section_{name}.html" ladcp_has = any(c in ladcp_cast_nums for c in cast_nums) n_casts_sec = len(cast_nums) _sec_n_casts_list.append(n_casts_sec) sections.append( { "name": name, "description": grp_cfg.get("description", ""), "color": grp_cfg.get("color", "#1a3a5c"), "n_casts": n_casts_sec, "cast_range": compact_cast_list(cast_nums) if cast_nums else "—", "report_exists": report_path.exists(), "ladcp_has": ladcp_has, } ) if cast_pos: sec_lats = [cast_pos[c][0] for c in cast_nums if c in cast_pos] sec_lons = [cast_pos[c][1] for c in cast_nums if c in cast_pos] # Match the section page's key_cast ordering so the sections-map # polyline does not fold when the section is anchored to a distance # origin. (The cruise-track map on index.html keeps occupation order.) key_cfg = grp_cfg.get("key_cast") if key_cfg is not None and len(sec_lats) >= 2: _key_nums = expand_cast_numbers([key_cfg]) _kn = _key_nums[0] if len(_key_nums) == 1 else None if _kn in cast_pos: _kl, _ko = cast_pos[_kn] _order = np.argsort( distance_from_km(_kl, _ko, sec_lats, sec_lons), kind="stable" ) sec_lats = [sec_lats[i] for i in _order] sec_lons = [sec_lons[i] for i in _order] if sec_lats: sections_data.append( { "name": name, "color": grp_cfg.get("color", "#555555"), "lats": sec_lats, "lons": sec_lons, } ) all_cast_lats = [v[0] for v in cast_pos.values()] all_cast_lons = [v[1] for v in cast_pos.values()] sections_map_b64 = ( _make_all_sections_map_b64( sections_data, all_cast_lats, all_cast_lons, target_h=3.0, cfg=cfg, width_in=_slot_width(_MAP_SLOT), ) if sections_data else None ) # Cruise-wide date range from all_meta _ci = cruise_info or {} _times_str = sorted( str(m["time_start"])[:10] for m in (all_meta or []) if m.get("time_start") ) _date_start = _times_str[0] if _times_str else "" _date_end = _times_str[-1] if len(_times_str) >= 2 else "" _duration_days = 0 if _date_start and _date_end: try: from datetime import date as _date _duration_days = ( _date.fromisoformat(_date_end) - _date.fromisoformat(_date_start) ).days + 1 except ValueError: pass # Summary ranges across sections _casts_range = "" if _sec_n_casts_list: mn, mx = min(_sec_n_casts_list), max(_sec_n_casts_list) _casts_range = f"{mn}–{mx}" if mn != mx else str(mn) ctx: dict[str, Any] = { "cruise": cruise, "ship": _ci.get("ship", ""), "date_start": _date_start, "date_end": _date_end, "duration_days": _duration_days, "casts_range": _casts_range, "sections": sections, "sections_map_b64": sections_map_b64, "map_slot": _MAP_SLOT, "ladcp_configured": ladcp_dir is not None, "version": _VERSION, "generated_at": report_generated_at(generated_at), } html = get_template("sections.html").render( **ctx, css=SHARED_CSS, js_top_links=_JS_TOP_LINKS, nav_prefix="", nav=nav_for("sections", prefix="", inventory=inventory_datasets or ()), ) (out_dir / "sections.html").write_text(html, encoding="utf-8") def _write_timeseries_list( timeseries_cfg: dict[str, Any], cruise: str, out_dir: Path, all_meta: list[dict[str, Any]] | None = None, ladcp_dir: Path | None = None, cruise_info: dict[str, Any] | None = None, inventory_datasets: list[dict[str, str]] | None = None, generated_at: str | None = None, cfg: ReportConfig = DEFAULT_REPORT_CONFIG, ) -> None: """Write timeseries.html listing all timeseries groups with an overview map.""" ladcp_cast_nums: set[int] = set() if ladcp_dir is not None and ladcp_dir.exists(): for f in ladcp_dir.glob("*.mat"): with contextlib.suppress(ValueError): ladcp_cast_nums.add(int(f.stem)) # Build cast → position and time lookup cast_pos: dict[int, tuple[float, float]] = {} cast_times: dict[int, tuple[Any, Any]] = {} if all_meta: for m in all_meta: cn = m.get("cast_num") if cn is not None: if np.isfinite(m.get("lat", np.nan)) and np.isfinite( m.get("lon", np.nan) ): cast_pos[int(cn)] = (float(m["lat"]), float(m["lon"])) if m.get("time_start") is not None and m.get("time_end") is not None: cast_times[int(cn)] = (m["time_start"], m["time_end"]) items = [] timeseries_data: list[dict[str, Any]] = [] _ts_n_casts_list: list[int] = [] _ts_dur_h_list: list[float] = [] for name, grp_cfg in timeseries_cfg.items(): cast_nums = expand_cast_numbers(grp_cfg.get("cast_numbers", [])) report_path = out_dir / "timeseries" / f"timeseries_{name}.html" color = grp_cfg.get("color", "#7b2d8b") ladcp_has = any(c in ladcp_cast_nums for c in cast_nums) n_casts_ts = len(cast_nums) _ts_n_casts_list.append(n_casts_ts) # Compute per-group duration from cast timestamps grp_starts = [cast_times[c][0] for c in cast_nums if c in cast_times] grp_ends = [cast_times[c][1] for c in cast_nums if c in cast_times] if grp_starts and grp_ends: try: _t0 = np.min(np.array(grp_starts, dtype="datetime64[ns]")) _t1 = np.max(np.array(grp_ends, dtype="datetime64[ns]")) _dur_h = float((_t1 - _t0) / np.timedelta64(1, "h")) _ts_dur_h_list.append(_dur_h) except Exception: # noqa: BLE001 pass items.append( { "name": name, "description": grp_cfg.get("description", ""), "color": color, "n_casts": n_casts_ts, "cast_range": compact_cast_list(cast_nums) if cast_nums else "—", "report_exists": report_path.exists(), "ladcp_has": ladcp_has, } ) if cast_pos: ts_lats = [cast_pos[c][0] for c in cast_nums if c in cast_pos] ts_lons = [cast_pos[c][1] for c in cast_nums if c in cast_pos] if ts_lats: timeseries_data.append( {"name": name, "color": color, "lats": ts_lats, "lons": ts_lons} ) all_cast_lats = [v[0] for v in cast_pos.values()] all_cast_lons = [v[1] for v in cast_pos.values()] timeseries_map_b64 = ( _make_all_sections_map_b64( timeseries_data, all_cast_lats, all_cast_lons, target_h=3.0, cfg=cfg, width_in=_slot_width(_MAP_SLOT), ) if timeseries_data else None ) # Cruise-wide date range _ci = cruise_info or {} _times_str = sorted( str(m["time_start"])[:10] for m in (all_meta or []) if m.get("time_start") ) _date_start = _times_str[0] if _times_str else "" _date_end = _times_str[-1] if len(_times_str) >= 2 else "" _duration_days = 0 if _date_start and _date_end: try: from datetime import date as _date _duration_days = ( _date.fromisoformat(_date_end) - _date.fromisoformat(_date_start) ).days + 1 except ValueError: pass # Summary ranges across timeseries groups _ts_casts_range = "" if _ts_n_casts_list: mn, mx = min(_ts_n_casts_list), max(_ts_n_casts_list) _ts_casts_range = f"{mn}–{mx}" if mn != mx else str(mn) _ts_dur_range = "" if _ts_dur_h_list: mn_h, mx_h = min(_ts_dur_h_list), max(_ts_dur_h_list) _ts_dur_range = f"{mn_h:.0f}–{mx_h:.0f} h" if mn_h != mx_h else f"{mn_h:.0f} h" ctx: dict[str, Any] = { "cruise": cruise, "ship": _ci.get("ship", ""), "date_start": _date_start, "date_end": _date_end, "duration_days": _duration_days, "ts_casts_range": _ts_casts_range, "ts_dur_range": _ts_dur_range, "timeseries": items, "timeseries_map_b64": timeseries_map_b64, "map_slot": _MAP_SLOT, "ladcp_configured": ladcp_dir is not None, "version": _VERSION, "generated_at": report_generated_at(generated_at), } html = get_template("timeseries.html").render( **ctx, css=SHARED_CSS, js_top_links=_JS_TOP_LINKS, nav_prefix="", nav=nav_for("timeseries", prefix="", inventory=inventory_datasets or ()), ) (out_dir / "timeseries.html").write_text(html, encoding="utf-8") # --------------------------------------------------------------------------- # Mtime helpers # --------------------------------------------------------------------------- def _fmt_mtime(path: Path) -> str: """Return a compact local datetime string for *path*'s mtime, or ``"missing"``.""" try: return ( datetime.fromtimestamp(path.stat().st_mtime, tz=UTC) .astimezone() .strftime("%Y-%m-%d %H:%M") ) except OSError: return "missing" def _mtime_skip_reason( expected: Path, source: Path, force: bool, *extra_sources: Path, skip_existing: bool = False, ) -> str: """Return a non-empty skip-reason string if *expected* should not be regenerated. Returns an empty string when the page should be (re)generated. Logic (evaluated in order): - *force* is True → always regenerate (return ``""``). - *expected* does not exist → must generate (return ``""``). - *skip_existing* is True → skip unconditionally: ``"skipped (exists)"``. - *expected* is newer than ALL of *source* and *extra_sources* → skip: ``"skipped (up to date)"``. - Any source newer than *expected* → regenerate (smart update, return ``""``). - mtime comparison fails (``OSError``) → conservative skip: ``"skipped (exists, use --force)"``. """ if force or not expected.exists(): return "" if skip_existing: return "skipped (exists)" try: html_mtime = expected.stat().st_mtime all_sources = [source, *extra_sources] if any(s.stat().st_mtime > html_mtime for s in all_sources): return "" # at least one source is newer: regenerate return "skipped (up to date)" except OSError: return "skipped (exists, use --force)" # --------------------------------------------------------------------------- # Metadata helpers # --------------------------------------------------------------------------- def _select_cast_files(nc_dir: Path) -> list[Path]: """Return the best-available per-cast file, sorted by cast identity. Each cast contributes its highest-available stage file (stage 3 > 2 > 1) from the stage layout under *nc_dir*; an old flat directory of unsuffixed files is read as stage 1 via the shim in :mod:`ctdcast.processors.stage_layout`. A plain cast ``NNN`` and its lettered sibling ``NNNb`` are distinct events. Sort order is cast number then suffix (plain before ``b``). """ return [path for _cast_id, path, _stage in select_best_available(nc_dir)] def _cast_qc_pct(nc_path: Path | None) -> float | None: """Return the percent of samples flagged suspect or fail for a cast, or None. Aggregates the QARTOD flag counts across every QC'd variable in the cast's file into one figure for the casts index — flags 3 (suspect) and 4 (fail); pass, not-evaluated and missing do not count. Returns ``None`` when the file carries no ``_qc`` companions (e.g. a stage-1 cast), rendered as ``–``. """ if nc_path is None: return None try: with xr.open_dataset(nc_path, engine="netcdf4", decode_timedelta=False) as ds: rows, _ = nc_tables.qc_flag_rows(ds, is_qc=is_qc) except (OSError, ValueError, KeyError): return None if not rows: return None total = sum(r["total"] for r in rows) flagged = sum(f["n"] for r in rows for f in r["flags"] if f["flag"] in (3, 4)) return round(100.0 * flagged / total, 1) if total else None def _read_cast_meta(nc_path: Path) -> dict[str, Any] | None: """Read scalar metadata from a cast .nc file without loading all data.""" try: # Context-managed so the handle always closes -- on an early return or an # exception as well as the success path. A leaked netCDF handle locks the # file on Windows (HDF5 file locking), blocking the next open of the same # cast (e.g. qc_summary) indefinitely. with xr.open_dataset(nc_path, decode_timedelta=False, engine="netcdf4") as ds: _id = cast_id_from_name(nc_path.stem) if _id is None: return None cast_num, cast_suffix = _id import warnings as _w with _w.catch_warnings(): _w.filterwarnings("ignore", "All-NaN slice encountered") lat = float(np.nanmedian(ds["latitude"].values)) lon = float(np.nanmedian(ds["longitude"].values)) max_depth = float(np.nanmax(ds["pressure"].values)) time_start = ds["time"].values[0] time_end = ds["time"].values[-1] raw_filename = ds.attrs.get("raw_filename", nc_path.stem + ".cnv") cruise = ds.attrs.get("cruise", UNKNOWN_CRUISE_ID) return { "cast_num": cast_num, "cast_suffix": cast_suffix, "cast_num_str": format_cast_id(cast_num, cast_suffix), "path": nc_path, "lat": lat, "lon": lon, "max_depth": max_depth, "time_start": time_start, "time_end": time_end, "raw_filename": raw_filename, "cruise": cruise, } except Exception: # noqa: BLE001 return None def _decode_str(value: Any) -> str: # noqa: ANN401 # netCDF scalar of caller-decided dtype (str/bytes/numpy scalar) """Return *value* as ``str``, decoding numpy/bytes scalars. netCDF string variables round-trip as ``bytes``/``numpy.bytes_`` under some writing libraries, so a plain ``str(value)`` would yield ``"b'b'"`` for a ``cast_suffix`` of ``b'b'`` — silently corrupting the cast id rather than failing loudly. """ item = value.item() if hasattr(value, "item") else value if isinstance(item, bytes): return item.decode("utf-8", "replace") return str(item) def _nat_safe_min(a: np.datetime64, b: np.datetime64) -> np.datetime64: """Return the earlier of two ``datetime64`` values, ignoring ``NaT``. ``NaT`` comparisons are always False, so a plain ``min()`` can return ``NaT`` or the wrong endpoint when one profile of a cast lacks a time. """ if np.isnat(a): return b if np.isnat(b): return a return min(a, b) def _nat_safe_max(a: np.datetime64, b: np.datetime64) -> np.datetime64: """Return the later of two ``datetime64`` values, ignoring ``NaT``.""" if np.isnat(a): return b if np.isnat(b): return a return max(a, b) def _read_meta_from_profiles(profiles_path: Path) -> list[dict[str, Any]]: """Build the per-cast metadata list from the compiled ``profiles.nc``. Used when no per-cast files are present. The index, cruise map and leaflet are *synthesis* views (per the design rule that station pages read per-cast stage files while sections, timeseries and the index read ``profiles.nc``), so their cast count, positions and times come from the compiled file — which records them exactly. This does **not** produce station pages: those exist to show raw-scan warts (soak, spikes, down/up mismatch) that binning averages away, so a station page from the grid would be defeated for its purpose. Returns one row per cast (the compiled file holds a down and an up profile per cast; they are merged — earliest start, latest end), matching the dict shape of :func:`_read_cast_meta` so downstream synthesis code is unchanged. """ try: ds = xr.open_dataset(profiles_path, decode_timedelta=False, engine="netcdf4") except Exception: # noqa: BLE001 return [] try: cruise = _decode_str(ds.attrs.get("cruise", UNKNOWN_CRUISE_ID)) nums = ds["cast_number"].values suffixes = ds["cast_suffix"].values if "cast_suffix" in ds else [""] * len(nums) lats = ds["latitude"].values lons = ds["longitude"].values t_start = ds["time_start"].values t_end = ds["time_end"].values maxp = ( ds["max_pressure_dbar"].values if "max_pressure_dbar" in ds else np.full(len(nums), np.nan) ) except KeyError: return [] finally: ds.close() by_cast: dict[tuple[int, str], dict[str, Any]] = {} for i in range(len(nums)): num, suffix = int(nums[i]), _decode_str(suffixes[i]) row = by_cast.get((num, suffix)) if row is None: by_cast[(num, suffix)] = { "cast_num": num, "cast_suffix": suffix, "cast_num_str": format_cast_id(num, suffix), "path": profiles_path, "lat": float(lats[i]), "lon": float(lons[i]), "max_depth": float(maxp[i]), "time_start": t_start[i], "time_end": t_end[i], "raw_filename": "", "cruise": cruise, } else: # Merge the down and up profiles of one cast into a single span, # tolerating a NaT time on either profile. row["time_start"] = _nat_safe_min(row["time_start"], t_start[i]) row["time_end"] = _nat_safe_max(row["time_end"], t_end[i]) return list(by_cast.values())