Generate Climos¶

This process runs generate_climos, which creates files with climatological means/standard deviations of input data from a netcdf file.

In [1]:
from birdy import WPSClient
from netCDF4 import Dataset
import os
import re
from wps_tools.testing import get_target_url
from wps_tools.output_handling import auto_construct_outputs, txt_to_string, get_metalink_content, nc_to_dataset
In [2]:
# NBVAL_IGNORE_OUTPUT
url = get_target_url("thunderbird")
print(f"Using thunderbird on {url}")

Using thunderbird on https://marble-dev01.pcic.uvic.ca/twitcher/ows/proxy/thunderbird/wps
In [3]:
thunderbird = WPSClient(url)
In [4]:
# NBVAL_IGNORE_OUTPUT
# Check info on `generate_climos` process
thunderbird.generate_climos?

Signature:
thunderbird.generate_climos(
    netcdf,
    operation,
    dry_run=None,
    convert_longitudes=True,
    split_vars=True,
    split_intervals=True,
    loglevel='INFO',
    climo=None,
    resolutions=None,
)
Docstring:
Generate files containing climatological means from input files of daily, monthly, or yearly data that adhere to the PCIC metadata standard (and consequently to CMIP5 and CF standards).

Parameters
----------
netcdf : ComplexData:mimetype:`application/x-netcdf`, :mimetype:`application/x-ogc-dods`
    NetCDF file
operation : {'mean', 'std'}string
    Operation to perform on the datasets
climo : {'6190', '7100', '8110', '2020', '2050', '2080'}string
    Year ranges
resolutions : {'all', 'yearly', 'seasonal', 'monthly'}string
    Temporal Resolutions
convert_longitudes : boolean
    Transform longitude range from [0, 360) to [-180, 180)
split_vars : boolean
    Generate a separate file for each dependent variable in the file
split_intervals : boolean
    Generate a separate file for each climatological period
dry_run : boolean
    Checks file to ensure compatible with process
loglevel : {'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'}string
    Logging level

Returns
-------
output : ComplexData:mimetype:`application/metalink+xml; version=4.0`
    Metalink object between output files
dry_output : ComplexData:mimetype:`application/metalink+xml; version=4.0`
    Metalink object between dry output files
File:      ~/code/thunderbird/notebooks/</home/eyvorchuk/.cache/pypoetry/virtualenvs/thunderbird-7g6X3rbj-py3.10/lib/python3.10/site-packages/birdy/client/base.py-0>
Type:      method

Single File run¶

Dry Run - Checks file to ensure compatible with process

In [5]:
# Set up variables for thunderbird.generate_climos
daccs_host = os.getenv("DACCS_HOST", "marble-dev01.pcic.uvic.ca")
seasonal_opendap = f'https://{daccs_host}/twitcher/ows/proxy/thredds/dodsC/datasets/storage/data/projects/comp_support/daccs/test-data/fdd_seasonal_CanESM2_rcp85_r1i1p1_1951-2100.nc'
annual_opendap = f'https://{daccs_host}/twitcher/ows/proxy/thredds/dodsC/datasets/storage/data/projects/comp_support/daccs/test-data/gdd_annual_CanESM2_rcp85_r1i1p1_1951-2100.nc'
operation = 'mean'
climo = '6190'
resolutions = 'yearly'
dry_run = True

# Dry run process
dry_output = thunderbird.generate_climos(
    netcdf=seasonal_opendap, 
    operation=operation, 
    climo=climo, 
    resolutions=resolutions, 
    dry_run=dry_run
)

Access the output with auto_construct_outputs() or get_metalink_content() with txt_to_string() from wps_tools.output_handling

In [6]:
# NBVAL_IGNORE_OUTPUT
auto_construct_outputs(dry_output.get())
Out[6]:
["Dry Run\ngenerate_climos:\nINFO:dp.generate_climos:Processing: https://marble-dev01.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/storage/data/projects/comp_support/daccs/test-data/fdd_seasonal_CanESM2_rcp85_r1i1p1_1951-2100.nc\nINFO:dp.generate_climos:climo_periods: {'6190'}\nINFO:dp.generate_climos:project: CMIP5\nINFO:dp.generate_climos:institution: PCIC\nINFO:dp.generate_climos:model: CanESM2\nINFO:dp.generate_climos:emissions: historical, rcp85\nINFO:dp.generate_climos:run: r1i1p1\nINFO:dp.generate_climos:dependent_varnames: ['fdd']\nINFO:dp.generate_climos:time_resolution: seasonal\nINFO:dp.generate_climos:is_multi_year_mean: False\n"]
In [7]:
# NBVAL_IGNORE_OUTPUT
meta_content = get_metalink_content(dry_output.get()[0])
print(meta_content)
txt_content = txt_to_string(meta_content[0])
print(txt_content)

['https://marble-dev01.pcic.uvic.ca/wpsoutputs/11cdc88a-1c66-11ef-8399-0242ac120003/fdd_seasonal_CanESM2_rcp85_r1i1p1_1951-2100_dry.txt']
Dry Run
generate_climos:
INFO:dp.generate_climos:Processing: https://marble-dev01.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/storage/data/projects/comp_support/daccs/test-data/fdd_seasonal_CanESM2_rcp85_r1i1p1_1951-2100.nc
INFO:dp.generate_climos:climo_periods: {'6190'}
INFO:dp.generate_climos:project: CMIP5
INFO:dp.generate_climos:institution: PCIC
INFO:dp.generate_climos:model: CanESM2
INFO:dp.generate_climos:emissions: historical, rcp85
INFO:dp.generate_climos:run: r1i1p1
INFO:dp.generate_climos:dependent_varnames: ['fdd']
INFO:dp.generate_climos:time_resolution: seasonal
INFO:dp.generate_climos:is_multi_year_mean: False
In [8]:
expected_items = ['6190', 'CMIP5', 'PCIC', 'CanESM2', 'historical', 'rcp85', 'r1i1p1', 'fdd', 'seasonal']
for item in expected_items:
    assert item in txt_content

Normal Run

In [9]:
# generate climos
output = thunderbird.generate_climos(
    netcdf=seasonal_opendap, 
    operation=operation, 
    climo=climo, 
    resolutions=resolutions, 
    dry_run=False
)
In [10]:
""" Helper function to test netCDF file output -
Creating a 30 year average with this process given the parameters should squash the time 
dimension down from x (where x is the number of days in the input data) to 1 in the output data. 
"""
def test_nc_data(url):
    output_data = nc_to_dataset(url)
    assert output_data.dimensions['time'].size == 1
In [11]:
# Test normal output data
url = get_metalink_content(output.get()[0])
test_nc_data(url[0])

Multiple File Run¶

Dry Run - Checks files to ensure compatible with process

In [12]:
# process dry run for multiple files
dry_output = thunderbird.generate_climos(
    netcdf=[seasonal_opendap, annual_opendap], 
    operation=operation, 
    climo=climo, 
    resolutions=resolutions, 
    dry_run=dry_run
)
In [13]:
# Test dry output for multiple files
metalinks = get_metalink_content(dry_output.get()[0])
assert len(metalinks) == 2
    
for link, tr in zip(metalinks, ['seasonal', 'yearly']):
    output_data = txt_to_string(link)
    assert re.search(r'time_resolution: {}'.format(tr), output_data)

Normal Run

In [14]:
# Process normal output for multiple files

output = thunderbird.generate_climos(
    netcdf=[seasonal_opendap,annual_opendap], 
    operation=operation, 
    climo=climo, 
    resolutions=resolutions, 
    dry_run=False
)
In [15]:
# Test multiple files
metalinks = get_metalink_content(output.get()[0])
assert len(metalinks) == 2
for url in metalinks:
    test_nc_data(url)