Parsing data from the Sonardyne FETCH AZA

The purpose of this notebook is to demonstrate the functionality of fetchAZA python package.

The demo is organised to show

• Step 1: Reading the *.csv files into xarray datasets

• Step 2: Writing the xarray datasets into individual netCDF files

• Step 3: Various plots

Note that when you submit a pull request, you should clear all outputs from your python notebook for a cleaner merge.

import pathlib
import sys

script_dir = pathlib.Path().parent.absolute()
parent_dir = script_dir.parents[0]
sys.path.append(str(parent_dir))

import xarray as xr
import os
import numpy as np
import matplotlib.pyplot as plt
import importlib
import datetime
from fetchAZA import convertAZA, readers, writers, plotters, tools, timetools, utilities
import warnings
import re
import glob
import logging
_log = logging.getLogger(__name__)

# Specify the path for writing datafiles
data_path = os.path.join(parent_dir, 'data')
fig_path = os.path.join(parent_dir, 'figures')

warnings.filterwarnings("ignore", message="In a future version of xarray decode_timedelta will default to False rather than None. To silence this warning, set decode_timedelta to True, False, or a 'CFTimedeltaCoder' instance.")
warnings.filterwarnings("ignore", category=xr.SerializationWarning, message="SerializationWarning: Can't decode floating point timedelta to 's' without precision loss, decoding to 'ns' instead. To silence this warning use time_unit='ns' in call to decoding function.")

Step 1 & 2 as convertAZA.convertAZA

fn = 'sample_data.csv'
STN = 'sample'
deploy_date = '2023-02-27'
recovery_date = '2023-03-08T08:00:00'
latitude = 26.5
longitude = -76.75
water_depth = -3800

print("Calling convertAZA with overwrite=True...")
print("This should NOT prompt for overwrite confirmation.")

# Force reload the module to pick up any changes
import importlib
importlib.reload(convertAZA)

# Test the overwrite parameter directly first
print(f"convertAZA function defaults: {convertAZA.convertAZA.__defaults__}")

ds_pressure, ds_AZA = convertAZA.convertAZA(
    data_path,
    fn,
    STN,
    deploy_date,
    recovery_date,
    latitude,
    longitude,
    water_depth,
    cleanup=True,
    overwrite=True
)

print("Function completed successfully!")

Calling convertAZA with overwrite=True...
This should NOT prompt for overwrite confirmation.
convertAZA function defaults: ('sample', '2000-01-01', '2099-01-01', '0', '0', '0', ['DQZ', 'PIES', 'INC', 'TMP', 'KLR'], True, None)

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for join will change from join='outer' to join='exact'. This change will result in the following ValueError: cannot be aligned with join='exact' because index/labels/sizes are not equal along these coordinates (dimensions): 'new_index' ('new_index',) The recommendation is to set join explicitly for this case.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))

/home/runner/work/fetchAZA/fetchAZA/data/sample_data*.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_KLR.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_DQZ.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_AZAseq.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_PIES.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_TMP.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_INC.nc
Dataset AZAseq not included in combined datasets

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/tools.py:353: FutureWarning: In a future version of xarray the default value for join will change from join='outer' to join='exact'. This change will result in the following ValueError: cannot be aligned with join='exact' because index/labels/sizes are not equal along these coordinates (dimensions): 'RECORD_TIME' ('RECORD_TIME',) The recommendation is to set join explicitly for this case.
  combined_dataset = xr.merge(combined_datasets.values(), compat="override")

Deleting file: /home/runner/work/fetchAZA/fetchAZA/data/sample_data_KLR.nc
Deleting file: /home/runner/work/fetchAZA/fetchAZA/data/sample_data_DQZ.nc
Deleting file: /home/runner/work/fetchAZA/fetchAZA/data/sample_data_PIES.nc
Deleting file: /home/runner/work/fetchAZA/fetchAZA/data/sample_data_TMP.nc
Deleting file: /home/runner/work/fetchAZA/fetchAZA/data/sample_data_INC.nc
Function completed successfully!

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/writers.py:129: UserWarning: Times can't be serialized faithfully to int64 with requested units 'seconds since 1970-01-01'. Resolution of 'milliseconds' needed. Serializing times to floating point instead. Set encoding['dtype'] to integer dtype to serialize to int64. Set encoding['dtype'] to floating point dtype to silence this warning.
  ds.to_netcdf(output_file)

Step 1: Read the *csv file for Logging Events.

This is done with readers.process_csv_to_xarray(). All logging events are read into individual xarray datasets, stored as a dictionary of datasets where the key. In addition, the AZA sequence (events following the pattern AZS-AZA-AZA-AZA-AZS) are read into an additional dataset with key ‘AZAseq’. Since this dataset does not contain every individual AZA or AZS event, it does not replace the individual datasets.

A log of the processing is also generated.

Optionally, the deployment and recovery dates can be passed. If they are, then the datasets will be sliced to these dates.

fn = 'sample_data.csv'
STN = 'sample'
deploy_date = '2023-02-27'
recovery_date = '2023-03-08T08:00:00'

# Process filename
file_path = os.path.join(data_path, fn)
file_root = fn.split('.')[0]
platform_id = file_root
today = datetime.datetime.now()
start_time = today.strftime("%Y%m%dT%H")

# Create a log file
log_file = os.path.join(data_path, f"{platform_id}_{start_time}_read.log")
logf_with_path = os.path.join(data_path, log_file)
logging.basicConfig(
    filename=logf_with_path,
    encoding='utf-8',
    format="%(asctime)s %(levelname)-8s %(funcName)s %(message)s",
    filemode="w", # 'w' to overwrite, 'a' to append
    level=logging.INFO,
    datefmt="%Y%m%dT%H%M%S",
    force=True,
    )
_log.info('Reading AZA from CSV to netCDF')
_log.info('Processing data from: %s', file_path)


# Process the CSV file and create xarray datasets containing the data
datasets = readers.read_csv_to_xarray(file_path)

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for join will change from join='outer' to join='exact'. This change will result in the following ValueError: cannot be aligned with join='exact' because index/labels/sizes are not equal along these coordinates (dimensions): 'new_index' ('new_index',) The recommendation is to set join explicitly for this case.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))
/home/runner/work/fetchAZA/fetchAZA/fetchAZA/readers.py:375: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.
  combined_dataset = xr.merge(list(datasets.values()))

Step 2: Write the data to netCDF

writers.save_datasets(datasets, file_path, overwrite=True)

Step 3: Further processing of pressure records and AZA sequence records

Note that in the steps above, the original data were not changed, with the exception of changes noted in the log file.

This means that each of the newly created *.nc mirrors–almost exactly–the original data.

Here we carry out additional steps including:

1. Load netCDF datasets based on provided keys. (readers.load_netcdf_datasets(data_path, file_root, keys))

2. Convert units and adjust time formats. (timetools.convert_seconds_to_float(ds))

3. Assign sampling time for the AZA sequence dataset. (timetools.assign_sample_time())

4. Filter datasets to the deployment period. (timetools.cut_to_deployment(datasets, deploy_date, recovery_date))

5. Reindex datasets on time. (timetools.reindex_on_time(ds))

6. Rename variables in datasets using predefined mappings. (using vars_to_rename, a dict)

7. Add dataset-specific attributes to variables.

8. Combine selected datasets into a single dataset. (using xr.merge())

9. Interpolate the combined dataset to an evenly spaced time grid. (after determining median interval of hourly with timetools.calculate_sample_rate(ds))

10. Clean and organize dataset attributes and variables.

11. Process the AZA sequence dataset, including renaming attributes and cleaning variables.

ds_pressure, ds_AZA = tools.process_datasets(data_path, file_root, deploy_date, recovery_date)
ds_pressure

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/tools.py:353: FutureWarning: In a future version of xarray the default value for join will change from join='outer' to join='exact'. This change will result in the following ValueError: cannot be aligned with join='exact' because index/labels/sizes are not equal along these coordinates (dimensions): 'RECORD_TIME' ('RECORD_TIME',) The recommendation is to set join explicitly for this case.
  combined_dataset = xr.merge(combined_datasets.values(), compat="override")

/home/runner/work/fetchAZA/fetchAZA/data/sample_data*.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_KLR.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_DQZ.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_AZAseq.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_PIES.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_TMP.nc
/home/runner/work/fetchAZA/fetchAZA/data/sample_data_INC.nc
Dataset AZAseq not included in combined datasets

<xarray.Dataset> Size: 7kB
Dimensions:               (TIME: 103)
Coordinates:
  * TIME                  (TIME) datetime64[ns] 824B 2023-03-03T23:00:00 ... ...
Data variables: (12/14)
    TIME_OF_FLIGHT_RMLE   (TIME) float64 824B 5.11 5.11 5.11 ... 5.148 5.146
    TIME_OF_FLIGHT_XCORR  (TIME) float64 824B 5.11 5.11 5.109 ... 5.109 5.109
    HALFLIFE              (TIME) float32 412B 251.4 230.3 229.1 ... 345.3 335.9
    MAGNITUDE             (TIME) float32 412B 6.903e+03 9.485e+03 ... 1.506e+03
    PEAK_POSITION         (TIME) float32 412B 0.008512 0.008704 ... 0.007808
    PITCH                 (TIME) float32 412B -1.875 -1.862 ... -1.853 -1.847
    ...                    ...
    PRETRIGGER_NOISE      (TIME) float32 412B 82.88 82.62 86.51 ... 64.36 65.5
    RDIFF                 (TIME) float32 412B 178.6 183.0 169.2 ... 86.83 89.4
    ROLL                  (TIME) float32 412B -0.1813 -0.1906 ... -0.1844
    TEMPERATURE           (TIME) float32 412B 3.073 2.9 2.756 ... 2.372 2.378
    TEMPERATURE_DQZ       (TIME) float32 412B 5.63 4.87 4.27 ... 2.43 2.43 2.43
    TEMPERATURE_KLR       (TIME) float32 412B 7.012 5.885 4.936 ... 2.215 2.214
Attributes:
    Calculation_Version_PIES:  3
    Index_DQZ:                 3
    Index_INC:                 1
    Index_KLR:                 5
    Index_PIES:                3
    Index_TMP:                 4
    Serial Number:             1262636
    Serial_Number_DQZ:         155399
    Serial_Number_INC:         0
    Serial_Number_KLR:         1262636
    Serial_Number_TMP:         0
    UID:                       007217

xarray.Dataset

• Dimensions:
  • TIME: 103
• Coordinates: (1)
  • TIME
    (TIME)
    datetime64[ns]
    2023-03-03T23:00:00 ... 2023-03-...

    array(['2023-03-03T23:00:00.000000000', '2023-03-04T00:00:00.000000000',
           '2023-03-04T01:00:00.000000000', '2023-03-04T02:00:00.000000000',
           '2023-03-04T03:00:00.000000000', '2023-03-04T04:00:00.000000000',
           '2023-03-04T05:00:00.000000000', '2023-03-04T06:00:00.000000000',
           '2023-03-04T07:00:00.000000000', '2023-03-04T08:00:00.000000000',
           '2023-03-04T09:00:00.000000000', '2023-03-04T10:00:00.000000000',
           '2023-03-04T11:00:00.000000000', '2023-03-04T12:00:00.000000000',
           '2023-03-04T13:00:00.000000000', '2023-03-04T14:00:00.000000000',
           '2023-03-04T15:00:00.000000000', '2023-03-04T16:00:00.000000000',
           '2023-03-04T17:00:00.000000000', '2023-03-04T18:00:00.000000000',
           '2023-03-04T19:00:00.000000000', '2023-03-04T20:00:00.000000000',
           '2023-03-04T21:00:00.000000000', '2023-03-04T22:00:00.000000000',
           '2023-03-04T23:00:00.000000000', '2023-03-05T00:00:00.000000000',
           '2023-03-05T01:00:00.000000000', '2023-03-05T02:00:00.000000000',
           '2023-03-05T03:00:00.000000000', '2023-03-05T04:00:00.000000000',
           '2023-03-05T05:00:00.000000000', '2023-03-05T06:00:00.000000000',
           '2023-03-05T07:00:00.000000000', '2023-03-05T08:00:00.000000000',
           '2023-03-05T09:00:00.000000000', '2023-03-05T10:00:00.000000000',
           '2023-03-05T11:00:00.000000000', '2023-03-05T12:00:00.000000000',
           '2023-03-05T13:00:00.000000000', '2023-03-05T14:00:00.000000000',
           '2023-03-05T15:00:00.000000000', '2023-03-05T16:00:00.000000000',
           '2023-03-05T17:00:00.000000000', '2023-03-05T18:00:00.000000000',
           '2023-03-05T19:00:00.000000000', '2023-03-05T20:00:00.000000000',
           '2023-03-05T21:00:00.000000000', '2023-03-05T22:00:00.000000000',
           '2023-03-05T23:00:00.000000000', '2023-03-06T00:00:00.000000000',
           '2023-03-06T01:00:00.000000000', '2023-03-06T02:00:00.000000000',
           '2023-03-06T03:00:00.000000000', '2023-03-06T04:00:00.000000000',
           '2023-03-06T05:00:00.000000000', '2023-03-06T06:00:00.000000000',
           '2023-03-06T07:00:00.000000000', '2023-03-06T08:00:00.000000000',
           '2023-03-06T09:00:00.000000000', '2023-03-06T10:00:00.000000000',
           '2023-03-06T11:00:00.000000000', '2023-03-06T12:00:00.000000000',
           '2023-03-06T13:00:00.000000000', '2023-03-06T14:00:00.000000000',
           '2023-03-06T15:00:00.000000000', '2023-03-06T16:00:00.000000000',
           '2023-03-06T17:00:00.000000000', '2023-03-06T18:00:00.000000000',
           '2023-03-06T19:00:00.000000000', '2023-03-06T20:00:00.000000000',
           '2023-03-06T21:00:00.000000000', '2023-03-06T22:00:00.000000000',
           '2023-03-06T23:00:00.000000000', '2023-03-07T00:00:00.000000000',
           '2023-03-07T01:00:00.000000000', '2023-03-07T02:00:00.000000000',
           '2023-03-07T03:00:00.000000000', '2023-03-07T04:00:00.000000000',
           '2023-03-07T05:00:00.000000000', '2023-03-07T06:00:00.000000000',
           '2023-03-07T07:00:00.000000000', '2023-03-07T08:00:00.000000000',
           '2023-03-07T09:00:00.000000000', '2023-03-07T10:00:00.000000000',
           '2023-03-07T11:00:00.000000000', '2023-03-07T12:00:00.000000000',
           '2023-03-07T13:00:00.000000000', '2023-03-07T14:00:00.000000000',
           '2023-03-07T15:00:00.000000000', '2023-03-07T16:00:00.000000000',
           '2023-03-07T17:00:00.000000000', '2023-03-07T18:00:00.000000000',
           '2023-03-07T19:00:00.000000000', '2023-03-07T20:00:00.000000000',
           '2023-03-07T21:00:00.000000000', '2023-03-07T22:00:00.000000000',
           '2023-03-07T23:00:00.000000000', '2023-03-08T00:00:00.000000000',
           '2023-03-08T01:00:00.000000000', '2023-03-08T02:00:00.000000000',
           '2023-03-08T03:00:00.000000000', '2023-03-08T04:00:00.000000000',
           '2023-03-08T05:00:00.000000000'], dtype='datetime64[ns]')

• Data variables: (14)
  • TIME_OF_FLIGHT_RMLE
    (TIME)
    float64
    5.11 5.11 5.11 ... 5.148 5.146

    units :

    seconds

    Logging Event :

    PIES

    array([5.110247, 5.109858, 5.110258, 5.110012, 5.110123, 5.110325,
           5.109411, 5.110261, 5.111422, 5.110181, 5.111349, 5.112188,
           5.110297, 5.110992, 5.112417, 5.109969, 5.108989, 5.110755,
           5.109984, 5.109122, 5.111227, 5.109663, 5.111249, 5.112476,
           5.111055, 5.111772, 5.155913, 5.160521, 5.148868, 5.144632,
           5.13858 , 5.147809, 5.13775 , 5.152843, 5.132422, 5.143715,
           5.13986 , 5.161743, 5.144659, 5.130815, 5.133262, 5.133424,
           5.13718 , 5.123191, 5.120764, 5.119447, 5.1192  , 5.119008,
           5.117986, 5.126578, 5.146705, 5.152339, 5.11192 , 5.140642,
           5.138413, 5.142701, 5.144412, 5.134315, 5.141353, 5.145822,
           5.11119 , 5.139293, 5.147685, 5.14918 , 5.108171, 5.145741,
           5.143932, 5.110996, 5.110052, 5.150368, 5.110949, 5.157961,
           5.141894, 5.110121, 5.112396, 5.11008 , 5.160424, 5.142199,
           5.110136, 5.109221, 5.108356, 5.138449, 5.138167, 5.137346,
           5.113582, 5.156992, 5.144918, 5.154339, 5.139468, 5.151501,
           5.111321, 5.166002, 5.110594, 5.158479, 5.166721, 5.111527,
           5.15003 , 5.145074, 5.190253, 5.14652 , 5.148196, 5.148328,
           5.145778])

  • TIME_OF_FLIGHT_XCORR
    (TIME)
    float64
    5.11 5.11 5.109 ... 5.109 5.109

    units :

    seconds

    Logging Event :

    PIES

    array([5.109885, 5.109944, 5.108936, 5.108754, 5.108865, 5.108939,
           5.109433, 5.108491, 5.109588, 5.109755, 5.111883, 5.110482,
           5.109871, 5.110054, 5.110455, 5.109351, 5.108947, 5.108921,
           5.109302, 5.108888, 5.109393, 5.109749, 5.109927, 5.11077 ,
           5.111397, 5.110002, 5.108767, 5.109535, 5.101658, 5.108622,
           5.108522, 5.109239, 5.109804, 5.109729, 5.109724, 5.110585,
           5.11025 , 5.107045, 5.110313, 5.109653, 5.109668, 5.108934,
           5.108466, 5.108557, 5.108754, 5.109293, 5.109558, 5.109686,
           5.109944, 5.110216, 5.109991, 5.109481, 5.109382, 5.108344,
           5.106499, 5.108035, 5.108146, 5.108545, 5.108799, 5.109236,
           5.109292, 5.108851, 5.102267, 5.109522, 5.104161, 5.108195,
           5.108178, 5.10545 , 5.107642, 5.10847 , 5.109435, 5.108639,
           5.108572, 5.109311, 5.109026, 5.108886, 5.108606, 5.105357,
           5.107534, 5.107835, 5.10761 , 5.109031, 5.107149, 5.110872,
           5.110916, 5.110358, 5.110188, 5.109433, 5.10909 , 5.108515,
           5.108463, 5.108104, 5.107672, 5.107557, 5.108567, 5.106045,
           5.109284, 5.109896, 5.103683, 5.108526, 5.108858, 5.109438,
           5.108744])

  • HALFLIFE
    (TIME)
    float32
    251.4 230.3 229.1 ... 345.3 335.9

    Logging Event :

    PIES

    array([251.39337, 230.27922, 229.05432, 248.44684, 234.31633, 223.24167,
           265.86505, 225.23824, 241.59631, 269.30753, 246.37376, 217.66837,
           201.66423, 242.16425, 241.72632, 200.69757, 220.8017 , 227.6382 ,
           201.65556, 207.28368, 211.43442, 190.46619, 213.33742, 211.62201,
           197.87833, 282.35852, 340.50632, 297.9106 ,       nan, 286.41885,
           304.20697, 253.5122 , 274.21133, 330.5854 , 301.5062 , 301.9266 ,
           364.73248, 295.71115, 279.38947, 289.75317, 275.951  , 322.69086,
           344.62943, 423.42874, 343.5263 , 585.1111 , 563.58124, 631.6579 ,
           764.4682 , 300.8514 , 338.49588, 349.19354, 310.83228, 277.50452,
           309.44293, 333.3152 , 325.51666, 312.68167, 316.3071 , 363.61905,
           393.8044 , 296.82358, 419.58353, 299.7992 , 389.6566 , 309.14133,
           292.95822, 356.35468, 310.84232, 387.91125, 334.09326, 305.6543 ,
           366.67804, 344.1589 , 316.1633 , 340.2852 , 360.43393, 410.54865,
           354.9099 , 374.13107, 363.07483, 304.57217, 346.44482, 325.46753,
           318.55197, 480.2876 , 368.67484, 367.19043, 355.31134, 348.7923 ,
           385.09866, 415.4711 , 309.2839 , 542.4069 , 317.23337, 348.1767 ,
           285.0195 , 316.94257, 291.65097, 329.4741 , 368.7936 , 345.25894,
           335.86673], dtype=float32)

  • MAGNITUDE
    (TIME)
    float32
    6.903e+03 9.485e+03 ... 1.506e+03

    Logging Event :

    PIES

    array([ 6903.,  9485.,  5926.,  7720.,  7467.,  5859.,  8928.,  6605.,
            8302.,  7399.,  9180., 11413.,  8459.,  7785.,  8392.,  8810.,
           10151.,  8812.,  6849.,  8794.,  7831., 12961.,  8641.,  8516.,
           14634.,  1709.,  1985.,  2300.,    nan,  2726.,  2315.,  3157.,
            2074.,  2000.,  1489.,  2543.,  2590.,  1255.,  2002.,  2644.,
            3835.,  4564.,  4474.,  7082.,  7565.,  8153., 10801.,  5641.,
           12190.,  4316.,  2463.,  2489.,  1690.,  3348.,  1365.,  1561.,
            2331.,  2970.,  1925.,  1957.,  1337.,  2014.,  1289.,  2838.,
            1165.,  2434.,  2236.,  1373.,  1633.,  3044.,  1613.,  1693.,
            1424.,  2268.,  1308.,  1419.,  1553.,  1130.,  1729.,  1775.,
            2137.,  2129.,  1345.,  2955.,  1550.,  1860.,  2438.,  1661.,
            1812.,  2269.,  1841.,  1826.,  1855.,  1861.,  1930.,  1124.,
            1831.,  2984.,  1537.,  1761.,  2249.,  1965.,  1506.],
          dtype=float32)

  • PEAK_POSITION
    (TIME)
    float32
    0.008512 0.008704 ... 0.007808

    Logging Event :

    PIES

    array([0.008512, 0.008704, 0.007872, 0.007872, 0.008128, 0.008256,
           0.008704, 0.007616, 0.008512, 0.008448, 0.010368, 0.008832,
           0.008192, 0.008448, 0.009024, 0.008128, 0.007936, 0.008064,
           0.008512, 0.008064, 0.008448, 0.00864 , 0.00864 , 0.009344,
           0.00992 , 0.008576, 0.007488, 0.008448,      nan, 0.007872,
           0.007808, 0.008448, 0.008832, 0.008512, 0.008256, 0.008896,
           0.008448, 0.005248, 0.00864 , 0.008192, 0.008448, 0.007936,
           0.007616, 0.007744, 0.007872, 0.008256, 0.00832 , 0.008256,
           0.008384, 0.00864 , 0.008512, 0.008192, 0.00832 , 0.007488,
           0.00576 , 0.007296, 0.007296, 0.007488, 0.007488, 0.00768 ,
           0.007552, 0.00704 , 0.000512, 0.007936, 0.002816, 0.007104,
           0.007296, 0.004672, 0.006848, 0.007552, 0.00832 , 0.007296,
           0.00704 , 0.00768 , 0.007424, 0.007424, 0.00736 , 0.004352,
           0.00672 , 0.007104, 0.006848, 0.008128, 0.006016, 0.009472,
           0.00928 , 0.008576, 0.008384, 0.007744, 0.007616, 0.007296,
           0.007488, 0.007296, 0.006912, 0.00672 , 0.007552, 0.0048  ,
           0.007808, 0.008256, 0.001984, 0.006912, 0.007424, 0.008256,
           0.007808], dtype=float32)

  • PITCH
    (TIME)
    float32
    -1.875 -1.862 ... -1.853 -1.847

    units :

    degrees

    observation_type :

    measured

    convention :

    forward positive

    resolution :

    0.0001

    long_name :

    Orientation (pitch) of measurement platform by inclinometer

    uri :

    https://vocab.nerc.ac.uk/collection/P02/current/PITCH/

    Logging Event :

    INC

    array([-1.875 , -1.8625, -1.8875, -1.8719, -1.8688, -1.8625, -1.8563,
           -1.8594, -1.85  , -1.8469, -1.8563, -1.8594, -1.8531, -1.8469,
           -1.85  , -1.85  , -1.8469, -1.85  , -1.85  , -1.8563, -1.8406,
           -1.8438, -1.85  , -1.8469, -1.85  , -1.8469, -1.8563, -1.8469,
               nan, -1.85  , -1.8531, -1.8438, -1.85  , -1.8531, -1.8563,
           -1.8406, -1.8563, -1.8656, -1.85  , -1.85  , -1.85  , -1.85  ,
           -1.85  , -1.8469, -1.8594, -1.8469, -1.8438, -1.85  , -1.8531,
           -1.8531, -1.8563, -1.8563, -1.8469, -1.8531, -1.875 , -1.85  ,
           -1.8563, -1.85  , -1.85  , -1.85  , -1.8563, -1.8469, -1.8563,
           -1.8531, -1.8531, -1.85  , -1.85  , -1.8469, -1.8625, -1.8563,
           -1.85  , -1.8531, -1.8531, -1.8531, -1.8531, -1.8563, -1.8469,
           -1.8469, -1.8563, -1.8531, -1.8406, -1.85  , -1.8625, -1.85  ,
           -1.8531, -1.85  , -1.8438, -1.8406, -1.8531, -1.8531, -1.8531,
           -1.8375, -1.8438, -1.8594, -1.8594, -1.8531, -1.85  , -1.8531,
           -1.8563, -1.8469, -1.85  , -1.8531, -1.8469], dtype=float32)

  • PRESSURE_DQZ
    (TIME)
    float32
    3.919e+03 3.919e+03 ... 3.919e+03

    units :

    dbar

    resolution :

    0.0001

    long_name :

    Pressure (spatial coordinate) exerted by the water body by pressure sensor

    observation_type :

    measured

    reference_datum :

    sea-surface

    standard_name :

    pressure

    Logging Event :

    DQZ

    comment :

    Interpolated pressure from DQZ, identical to pressure in PIES logging events

    array([3919.2722, 3919.1716, 3919.037 , 3918.8992, 3918.789 , 3918.7485,
           3918.7837, 3918.8938, 3919.0469, 3919.222 , 3919.3796, 3919.4824,
           3919.5044, 3919.449 , 3919.3164, 3919.1582, 3918.9968, 3918.8801,
           3918.8298, 3918.8552, 3918.947 , 3919.072 , 3919.207 , 3919.3123,
           3919.3508, 3919.3123, 3919.201 , 3919.0552,       nan, 3918.7988,
           3918.7715, 3918.8306, 3918.968 , 3919.154 , 3919.3445, 3919.5117,
           3919.5974, 3919.5938, 3919.5   , 3919.3386, 3919.156 , 3918.9875,
           3918.875 , 3918.847 , 3918.899 , 3919.0173, 3919.1692, 3919.3152,
           3919.4138, 3919.4263, 3919.3523, 3919.2087, 3919.036 , 3918.8796,
           3918.791 , 3918.7908, 3918.8748, 3919.032 , 3919.2246, 3919.4114,
           3919.5508, 3919.6047, 3919.5623, 3919.434 , 3919.2512, 3919.0579,
           3918.899 , 3918.82  , 3918.8325, 3918.9265, 3919.0762, 3919.2488,
           3919.393 , 3919.468 , 3919.4458, 3919.3398, 3919.1755, 3918.9922,
           3918.8477, 3918.7844, 3918.8083, 3918.9148, 3919.0896, 3919.2917,
           3919.472 , 3919.5828, 3919.5994, 3919.5122, 3919.3484, 3919.1555,
           3918.9705, 3918.843 , 3918.8064, 3918.865 , 3919.0005, 3919.175 ,
           3919.3503, 3919.4746, 3919.52  , 3919.4556, 3919.3184, 3919.1274,
           3918.9402], dtype=float32)

  • PRESSURE_KLR
    (TIME)
    float32
    3.919e+03 3.919e+03 ... 3.918e+03

    units :

    dbar

    resolution :

    0.0001

    long_name :

    Pressure (spatial coordinate) exerted by the water body by pressure sensor

    observation_type :

    measured

    reference_datum :

    sea-surface

    standard_name :

    pressure

    Logging Event :

    KLR

    array([3918.6445, 3918.615 , 3918.4082, 3918.242 , 3918.1172, 3918.0564,
           3918.082 , 3918.193 , 3918.345 , 3918.5037, 3918.6523, 3918.7363,
           3918.7563, 3918.696 , 3918.576 , 3918.403 , 3918.218 , 3918.106 ,
           3918.054 , 3918.068 , 3918.154 , 3918.274 , 3918.3992, 3918.4863,
           3918.5305, 3918.4978, 3918.4556, 3918.2317,       nan, 3917.9646,
           3917.9324, 3917.9724, 3918.1152, 3918.2969, 3918.4849, 3918.6404,
           3918.7297, 3918.7244, 3918.6216, 3918.4617, 3918.295 , 3918.0962,
           3917.9783, 3917.9343, 3917.994 , 3918.1082, 3918.268 , 3918.397 ,
           3918.5032, 3918.5125, 3918.4304, 3918.282 , 3918.1091, 3917.9453,
           3917.8618, 3917.8481, 3917.929 , 3918.0981, 3918.2732, 3918.4648,
           3918.604 , 3918.647 , 3918.602 , 3918.49  , 3918.2917, 3918.0955,
           3917.9348, 3917.8406, 3917.865 , 3917.949 , 3918.1   , 3918.2708,
           3918.4119, 3918.485 , 3918.5378, 3918.3652, 3918.1914, 3918.007 ,
           3917.8494, 3917.7917, 3917.8093, 3917.9204, 3918.0898, 3918.2898,
           3918.4778, 3918.5837, 3918.5994, 3918.5085, 3918.3352, 3918.153 ,
           3917.9536, 3917.8325, 3917.7927, 3917.8542, 3917.9817, 3918.1606,
           3918.3286, 3918.4426, 3918.4856, 3918.4197, 3918.287 , 3918.0867,
           3917.8962], dtype=float32)

  • PRETRIGGER_NOISE
    (TIME)
    float32
    82.88 82.62 86.51 ... 64.36 65.5

    Logging Event :

    PIES

    array([ 82.88018 ,  82.621605,  86.50508 ,  94.26092 ,  83.696724,
            77.835236,  89.09337 ,  95.13427 , 105.12323 ,  92.6603  ,
           100.88058 ,  92.77622 ,  87.56836 ,  92.46656 ,  84.74118 ,
            90.01213 ,  86.11518 ,  79.99684 ,  82.304085,  93.171036,
            88.30831 ,  78.20332 ,  76.161674,  80.762535,  76.51092 ,
            31.675522,  62.49573 ,  64.79453 ,        nan,  72.95018 ,
            65.42326 ,  62.82204 ,  70.279854,  64.48062 ,  66.95363 ,
            67.388885,  65.44143 ,  63.94122 ,  67.34427 ,  75.192924,
            72.81584 ,  73.88142 ,  77.813675,  91.12039 , 106.8119  ,
           100.52448 , 121.00624 ,  87.02094 , 128.16753 ,  82.348404,
            66.7866  ,  64.50473 ,  40.102627,  62.338825,  72.32161 ,
            64.579636,  66.95504 ,  60.788113,  64.60195 ,  61.17328 ,
            42.26049 ,  65.87446 ,  61.65703 ,  64.421974,  39.409218,
            65.35631 ,  58.99447 ,  43.88175 ,  41.959133,  63.117252,
            35.321575,  59.47024 ,  63.710354,  36.933136,  37.655994,
            36.954323,  60.311077,  64.35583 ,  44.654537,  41.738434,
            39.867058,  59.4815  ,  62.781498,  65.67463 ,  42.594505,
            55.929558,  67.21253 ,  61.824776,  67.618546,  62.65341 ,
            42.359272,  59.612343,  41.825146,  62.51128 ,  54.832718,
            41.704918,  68.94338 ,  69.66822 ,  55.290012,  66.84985 ,
            61.284344,  64.36308 ,  65.495255], dtype=float32)

  • RDIFF
    (TIME)
    float32
    178.6 183.0 169.2 ... 86.83 89.4

    Logging Event :

    PIES

    array([ 178.59473 ,  182.98535 ,  169.1748  ,  150.05371 ,  173.15137 ,
            220.28906 ,  140.60547 ,  167.81445 ,  135.52148 ,  143.47852 ,
            131.85938 ,  177.20508 ,  180.58496 ,  173.81543 ,  201.67773 ,
            172.16406 ,  173.69336 ,  200.98828 ,  208.23438 ,  160.7793  ,
            199.78711 ,  204.82031 ,  209.4961  ,  236.28809 ,  249.30176 ,
             94.43945 ,   82.32129 ,   90.59473 ,         nan,  141.44336 ,
             89.87891 ,  139.25293 ,  118.6084  ,  111.33789 ,   86.33594 ,
             98.08887 ,   90.2334  ,   83.59375 ,  107.171875,  147.4668  ,
            246.4834  ,  156.16602 ,  314.6211  ,  410.5879  ,  863.584   ,
            615.5869  , 1026.7119  ,  408.22168 , 1031.9463  ,  242.2168  ,
            119.28711 ,  100.55371 ,   83.42969 ,  136.03809 ,  104.58203 ,
             78.97461 ,   95.134766,   93.75586 ,   68.42578 ,   66.87109 ,
             62.916992,   75.85742 ,   78.86719 ,   61.251953,   61.90918 ,
             86.55859 ,   90.98242 ,   65.16699 ,   69.55664 ,   69.694336,
             81.15723 ,   73.87891 ,   93.80176 ,   73.822266,   78.228516,
             78.08301 ,   92.47656 ,   74.45215 ,   57.0791  ,   58.148438,
             71.06641 ,   84.29785 ,   70.45117 ,   75.18848 ,   76.146484,
             99.34082 ,   76.24805 ,   63.52539 ,   70.62891 ,   70.69531 ,
             70.83008 ,   70.143555,   72.47949 ,   75.75781 ,   90.87109 ,
             69.00879 ,  115.49902 ,  125.97949 ,  187.64844 ,   84.58789 ,
             66.583984,   86.8291  ,   89.39844 ], dtype=float32)

  • ROLL
    (TIME)
    float32
    -0.1813 -0.1906 ... -0.1969 -0.1844

    units :

    degrees

    observation_type :

    measured

    convention :

    port positive

    resolution :

    0.0001

    long_name :

    Orientation (roll angle) of measurement platform by inclinometer

    uri :

    https://vocab.nerc.ac.uk/collection/P02/current/ROLL/

    Logging Event :

    INC

    array([-0.1813, -0.1906, -0.1781, -0.1781, -0.1906, -0.1687, -0.1719,
           -0.2   , -0.1937, -0.1906, -0.1969, -0.1938, -0.1813, -0.1938,
           -0.1875, -0.1875, -0.1906, -0.1969, -0.2   , -0.1875, -0.1875,
           -0.1906, -0.2156, -0.1875, -0.2   , -0.1906, -0.2   , -0.2   ,
               nan, -0.1969, -0.1844, -0.1875, -0.2031, -0.1969, -0.1938,
           -0.1875, -0.1938, -0.2031, -0.1844, -0.1813, -0.1969, -0.1813,
           -0.1844, -0.1938, -0.1813, -0.1906, -0.1844, -0.1719, -0.1938,
           -0.1875, -0.1906, -0.1969, -0.1906, -0.1906, -0.1938, -0.1969,
           -0.1969, -0.1969, -0.1844, -0.1938, -0.1875, -0.1813, -0.1813,
           -0.1906, -0.1938, -0.1781, -0.1844, -0.1875, -0.1813, -0.1938,
           -0.1875, -0.1844, -0.1906, -0.1906, -0.1781, -0.1844, -0.1906,
           -0.1969, -0.1813, -0.1938, -0.1844, -0.1844, -0.1813, -0.1906,
           -0.2   , -0.1813, -0.1938, -0.1813, -0.1938, -0.1969, -0.1781,
           -0.1938, -0.2   , -0.1969, -0.1812, -0.1906, -0.1875, -0.1875,
           -0.1844, -0.1969, -0.1906, -0.1969, -0.1844], dtype=float32)

  • TEMPERATURE
    (TIME)
    float32
    3.073 2.9 2.756 ... 2.372 2.378

    Logging Event :

    TMP

    array([3.0726, 2.8997, 2.7557, 2.6579, 2.54  , 2.4765, 2.4282, 2.3975,
           2.3836, 2.372 , 2.3609, 2.3495, 2.3162, 2.2902, 2.2814, 2.2806,
           2.2897, 2.2951, 2.2868, 2.2847, 2.2847, 2.2884, 2.2878, 2.2897,
           2.2902, 2.2881, 2.2958,    nan, 2.293 , 2.293 , 2.2948, 2.2984,
           2.301 , 2.3067, 2.3118, 2.3131, 2.3129, 2.308 , 2.3067, 2.3064,
           2.3095, 2.3144, 2.3147, 2.3142, 2.3167, 2.3162, 2.3108, 2.3103,
           2.3134, 2.3118, 2.3147, 2.3165, 2.3271, 2.3219, 2.3173, 2.3214,
           2.3294, 2.3405, 2.3433, 2.3462, 2.35  , 2.3526, 2.3521, 2.3467,
           2.3482, 2.3462, 2.3384, 2.3389, 2.34  , 2.3387, 2.3384, 2.3397,
           2.3423, 2.3431, 2.3531, 2.3462, 2.3467, 2.3464, 2.3487, 2.349 ,
           2.3503, 2.3493, 2.349 , 2.3493, 2.35  , 2.356 , 2.3591, 2.3645,
           2.3681, 2.3663, 2.3699, 2.3815, 2.3722, 2.3696, 2.3758, 2.3782,
           2.381 , 2.3833, 2.3818, 2.3764, 2.3862, 2.372 , 2.3782],
          dtype=float32)

  • TEMPERATURE_DQZ
    (TIME)
    float32
    5.63 4.87 4.27 ... 2.43 2.43 2.43

    units :

    Celsius

    observation_type :

    measured

    standard_name :

    temperature

    resolution :

    0.001

    valid_max :

    42.0

    valid_min :

    -5.0

    uri :

    https://vocab.nerc.ac.uk/collection/OG1/current/TEMP/

    Logging Event :

    DQZ

    array([5.63, 4.87, 4.27, 3.84, 3.5 , 3.23, 3.03, 2.88, 2.77, 2.68, 2.62,
           2.57, 2.52, 2.47, 2.44, 2.41, 2.39, 2.39, 2.37, 2.37, 2.36, 2.35,
           2.35, 2.35, 2.35, 2.35, 2.39, 2.38,  nan, 2.36, 2.35, 2.35, 2.35,
           2.35, 2.36, 2.36, 2.36, 2.36, 2.36, 2.36, 2.36, 2.36, 2.36, 2.36,
           2.37, 2.37, 2.37, 2.36, 2.36, 2.36, 2.36, 2.37, 2.37, 2.37, 2.37,
           2.37, 2.37, 2.38, 2.39, 2.39, 2.39, 2.4 , 2.4 , 2.4 , 2.4 , 2.4 ,
           2.4 , 2.39, 2.39, 2.39, 2.39, 2.39, 2.39, 2.39, 2.43, 2.43, 2.42,
           2.41, 2.41, 2.4 , 2.4 , 2.4 , 2.4 , 2.4 , 2.4 , 2.4 , 2.41, 2.41,
           2.41, 2.42, 2.42, 2.42, 2.42, 2.42, 2.42, 2.43, 2.43, 2.43, 2.43,
           2.43, 2.43, 2.43, 2.43], dtype=float32)

  • TEMPERATURE_KLR
    (TIME)
    float32
    7.012 5.885 4.936 ... 2.215 2.214

    units :

    Celsius

    observation_type :

    measured

    standard_name :

    temperature

    resolution :

    0.001

    valid_max :

    42.0

    valid_min :

    -5.0

    uri :

    https://vocab.nerc.ac.uk/collection/OG1/current/TEMP/

    Logging Event :

    KLR

    array([7.012, 5.885, 4.936, 4.279, 3.786, 3.407, 3.123, 2.903, 2.741,
           2.613, 2.517, 2.441, 2.381, 2.32 , 2.272, 2.235, 2.208, 2.192,
           2.177, 2.166, 2.156, 2.149, 2.145, 2.141, 2.138, 2.134, 2.251,
           2.184,   nan, 2.155, 2.145, 2.142, 2.141, 2.14 , 2.143, 2.145,
           2.147, 2.146, 2.147, 2.146, 2.145, 2.149, 2.148, 2.149, 2.15 ,
           2.15 , 2.149, 2.149, 2.148, 2.148, 2.148, 2.149, 2.152, 2.155,
           2.155, 2.154, 2.157, 2.161, 2.165, 2.171, 2.176, 2.179, 2.182,
           2.184, 2.183, 2.184, 2.181, 2.177, 2.178, 2.179, 2.176, 2.175,
           2.177, 2.177, 2.286, 2.225, 2.207, 2.198, 2.192, 2.19 , 2.188,
           2.187, 2.185, 2.185, 2.184, 2.186, 2.188, 2.192, 2.196, 2.199,
           2.199, 2.204, 2.206, 2.204, 2.205, 2.208, 2.211, 2.213, 2.214,
           2.214, 2.215, 2.215, 2.214], dtype=float32)

• Indexes: (1)
  • TIME
    PandasIndex

    PandasIndex(DatetimeIndex(['2023-03-03 23:00:00', '2023-03-04 00:00:00',
                   '2023-03-04 01:00:00', '2023-03-04 02:00:00',
                   '2023-03-04 03:00:00', '2023-03-04 04:00:00',
                   '2023-03-04 05:00:00', '2023-03-04 06:00:00',
                   '2023-03-04 07:00:00', '2023-03-04 08:00:00',
                   ...
                   '2023-03-07 20:00:00', '2023-03-07 21:00:00',
                   '2023-03-07 22:00:00', '2023-03-07 23:00:00',
                   '2023-03-08 00:00:00', '2023-03-08 01:00:00',
                   '2023-03-08 02:00:00', '2023-03-08 03:00:00',
                   '2023-03-08 04:00:00', '2023-03-08 05:00:00'],
                  dtype='datetime64[ns]', name='TIME', length=103, freq=None))

• Attributes: (12)

  Calculation_Version_PIES :

  3

  Index_DQZ :

  3

  Index_INC :

  1

  Index_KLR :

  5

  Index_PIES :

  3

  Index_TMP :

  4

  Serial Number :

  1262636

  Serial_Number_DQZ :

  155399

  Serial_Number_INC :

  0

  Serial_Number_KLR :

  1262636

  Serial_Number_TMP :

  0

  UID :

  007217

Save the data

# Save the datasets
output_file = os.path.join(data_path, f"{STN}_{deploy_date.replace('-','')}_use.nc")
writers.save_dataset(ds_pressure, output_file, overwrite=True)

output_file = os.path.join(data_path, f"{STN}_{deploy_date.replace('-','')}_AZA.nc")
writers.save_dataset(ds_AZA, output_file, overwrite=True)

/home/runner/work/fetchAZA/fetchAZA/fetchAZA/writers.py:129: UserWarning: Times can't be serialized faithfully to int64 with requested units 'seconds since 1970-01-01'. Resolution of 'milliseconds' needed. Serializing times to floating point instead. Set encoding['dtype'] to integer dtype to serialize to int64. Set encoding['dtype'] to floating point dtype to silence this warning.
  ds.to_netcdf(output_file)

True

Plot variables in ds_pressure

# Example usage
plotters.plot_temperature_variables(ds_pressure, ['TEMPERATURE', 'PRESSURE'],"sample")

print("From the analysis, we determine that PRESSURE_DQZ and PRESSURE_PIES are identical.")

From the analysis, we determine that PRESSURE_DQZ and PRESSURE_PIES are identical.

Diagnostics & basic statistics

1. Plot variables which are against time

# Call the function to plot all variables against RECORD_TIME
#fig, axs = plotters.plot_all_variables_against_time(ds_pressure, time_var='RECORD_TIME')
fig,axs = plotters.plot_all_variables(ds_pressure, 'sample')

2. Histograms

# Example usage
plotters.plot_histograms(ds_pressure, "sample")

3. Compare pressure

Plot the pressure from the ambient (KLR) and transfer (DQZ)

# Example usage
fig, axs = plotters.compare_pressure(ds_pressure, ['PRESSURE_DQZ', 'PRESSURE_KLR'], "sample")

Plot the pressure during an AZA sequence (transfer, ambient and low)

variables_to_compare=['TRANSFER_PRESSURE','AMBIENT_PRESSURE','LOW_PRESSURE']

# Example usage
fig, axs = plotters.plot_AZA_pressure(ds_AZA, variables_to_compare)

# Plot all pressure variables in ds_AZA on the same axes with symbols and lines
plt.figure(figsize=(8, 4))
pressure_vars = ['TRANSFER_PRESSURE', 'AMBIENT_PRESSURE', 'LOW_PRESSURE']
time_var = 'SAMPLE_TIME'
# Define marker styles to cycle through
markers = ['o', 'v', 's', 'D', '^']

for idx, var in enumerate(pressure_vars):
    plt.plot(
        ds_AZA[time_var],
        ds_AZA[var],
        label=var,
        marker=markers[idx % len(markers)],
        linestyle='-',
        markerfacecolor='none'
    )

plt.title('Pressure Variables in AZA')
plt.xlabel('Time')
plt.ylabel('Pressure (kPa)')
plt.legend()
plt.grid(True)
plt.gca().invert_yaxis()  # Invert the y-axis
plt.tight_layout()
plt.show()

# Let's test the convertAZA function call to see what parameters are being passed
import sys
import pathlib
sys.path.append('/Users/eddifying/Cloudfree/gitlab-cloudfree/fetchAZA')

from fetchAZA import convertAZA

# Let's inspect the function signature
import inspect
sig = inspect.signature(convertAZA.convertAZA)
print("Function signature:")
print(sig)

# Let's see what the default values are
for name, param in sig.parameters.items():
    print(f"{name}: default = {param.default}")

Function signature:
(data_path, fn, STN='sample', deploy_date='2000-01-01', recovery_date='2099-01-01', latitude='0', longitude='0', water_depth='0', keys=['DQZ', 'PIES', 'INC', 'TMP', 'KLR'], cleanup=True, overwrite=None)
data_path: default = <class 'inspect._empty'>
fn: default = <class 'inspect._empty'>
STN: default = sample
deploy_date: default = 2000-01-01
recovery_date: default = 2099-01-01
latitude: default = 0
longitude: default = 0
water_depth: default = 0
keys: default = ['DQZ', 'PIES', 'INC', 'TMP', 'KLR']
cleanup: default = True
overwrite: default = None

# Let's test a simplified call to see what's happening
import os
data_path = '/Users/eddifying/Cloudfree/gitlab-cloudfree/fetchAZA/data'
fn = 'sample_data.csv'

# Test the function call that should work
print("Testing function call...")
print("This should set overwrite=True and not prompt:")

# Let's simulate the call that should not prompt
test_call = {
    'data_path': data_path,
    'fn': fn,
    'STN': 'sample',
    'deploy_date': '2023-02-27',
    'recovery_date': '2023-03-08T08:00:00',
    'latitude': 26.5,
    'longitude': -76.75,
    'water_depth': -3800,
    'cleanup': True,
    'overwrite': True
}

print("Parameters that will be passed:")
for key, value in test_call.items():
    print(f"  {key}: {value}")

Testing function call...
This should set overwrite=True and not prompt:
Parameters that will be passed:
  data_path: /Users/eddifying/Cloudfree/gitlab-cloudfree/fetchAZA/data
  fn: sample_data.csv
  STN: sample
  deploy_date: 2023-02-27
  recovery_date: 2023-03-08T08:00:00
  latitude: 26.5
  longitude: -76.75
  water_depth: -3800
  cleanup: True
  overwrite: True

# Let's test the save_dataset function directly to see if it's working
import sys
sys.path.append('/Users/eddifying/Cloudfree/gitlab-cloudfree/fetchAZA')

from fetchAZA import writers
import xarray as xr
import numpy as np
import tempfile
import os

# Create a simple test datasety
test_ds = xr.Dataset({
    'test_var': (['time'], np.random.rand(5))
}, coords={'time': np.arange(5)})

# Create a temporary file
with tempfile.NamedTemporaryFile(suffix='.nc', delete=False) as tmp:
    tmp_file = tmp.name

# Save it once
result1 = writers.save_dataset(test_ds, tmp_file, overwrite=True)
print(f"First save result: {result1}")
print(f"File exists: {os.path.exists(tmp_file)}")

# Try to save again with overwrite=True (should not prompt)
print("\nTesting overwrite=True (should not prompt)...")
result2 = writers.save_dataset(test_ds, tmp_file, overwrite=True)
print(f"Second save result: {result2}")

# Clean up
os.remove(tmp_file)
print("Test completed")

First save result: True
File exists: True

Testing overwrite=True (should not prompt)...
Second save result: True
Test completed