swarmpal.io

swarmpal.io#

Submodules#

Classes#

PalDataItem

An Item (i.e. dataset) that will be stored within a PalData object

PalDataTreeAccessor

Provide custom attributes and methods on XArray DataTrees for SwarmPAL functionality.

PalProcess

Abstract class to define processes to act on datatrees

Functions#

create_paldata(→ xarray.DataTree)

Generates a Datatree from a number of PalDataItems

Package Contents#

class swarmpal.io.PalDataItem(fetcher: swarmpal.io._datafetchers.DataFetcherBase)[source]#

An Item (i.e. dataset) that will be stored within a PalData object

Intended use is through the .from_… creator tools

Parameters:

fetcher (DataFetcherBase)

Examples

>>> from swarmpal.io import PalDataItem
>>>
>>> # Create the item with a configuration
>>> params = dict(
>>>     collection="SW_OPER_MAGA_LR_1B",
>>>     measurements=["F", "B_NEC"],
>>>     start_time="2016-01-01T00:00:00",
>>>     end_time="2016-01-02T00:00:00",
>>>     server_url="https://vires.services/ows",
>>> )
>>> item = PalDataItem.from_vires(**params)
>>> # "Initialise" - triggers the expensive part, fetching the data
>>> item.initialise()
>>> # Data is available as an xarray.Dataset
>>> item.xarray
>>> # or as a DataTree
>>> item.datatree
property xarray: xarray.Dataset | None#

xarray.Dataset containing the data, generated if not already present

_fetcher#
property dataset_name: str#

Name of the dataset, used as the datatree label

property datatree: xarray.DataTree#

Create a new datatree containing only this dataset; labelled with the dataset name.

property analysis_window: tuple[datetime.datetime]#

The start and end times of the analysis window (considering optional padding

property magnetic_models: dict#

Dictionary of model names and details

_serialise_pal_metadata()[source]#
initialise()[source]#

Trigger the fetching of the data and attach PAL metadata

static _ensure_datetime(times: tuple[datetime.datetime | str]) → tuple[datetime.datetime][source]#

Converts times to datetimes if they are not already

static _datetime_to_str(times: tuple[datetime.datetime]) → tuple[str][source]#

Convert datetime objects to iso strings

static _get_start_end_times(params: dict) → tuple[dict, tuple[datetime.datetime]][source]#

Get start and end times from the job parameters

Accounts for difference between VirES (“start_time”, “end_time”), and HAPI (“start”, “stop”)

static _update_start_end_times(params: dict, start: str, end: str)[source]#

Update the job parameters with new (start, end) times

static _pad_times(params: dict) → tuple[dict, tuple[datetime.datetime]][source]#

Use job parameters to adjust time window

Returns the modified params dictionary, as well as the analysis window

static from_vires(**params) → PalDataItem[source]#

Create PalDataItem from VirES source

Parameters:

  • collection (str)

  • measurements (list[str])

  • start_time (str | datetime)

  • end_time (str | datetime)

  • models (list[str])

  • auxiliaries (list[str])

  • sampling_step (str)

  • filters (list[str])

  • options (dict)

  • server_url (str) – defaults to “https://vires.services/ows”

  • pad_times (tuple[timedelta]) – This is handled specially by SwarmPAL and not passed to viresclient

static from_hapi(**params) → PalDataItem[source]#

Create PalDataItem from HAPI source

Parameters:

  • server (str)

  • dataset (str)

  • parameters (str)

  • start (str)

  • stop (str)

  • options (dict)

  • pad_times (tuple[timedelta]) – This is handled specially by SwarmPAL and not passed to hapiclient

static from_file(filename: os.PathLike | None = None, group: str | None = None, filetype: str = 'auto', **params) → PalDataItem[source]#

Create a PalDataItem from a file

Parameters:

  • filename (PathLike) – Path to the (netCDF / CDF) file to load

  • group (str) – Group name within the netCDF file

  • filetype (str) – Type of file to load. Can be “netcdf”, “cdf” or “auto”. If “auto”, the file extension will be used to determine the type.

  • params (dict) – Additional parameters to pass to the fetcher

Return type:

PalDataItem

static from_manual(xarray_dataset: xarray.Dataset | None = None, **params) → PalDataItem[source]#

Create a PalDataItem manually from an existing xarray Dataset

Parameters:

xarray_dataset (Dataset) – An existing xarray.Dataset

Return type:

PalDataItem

class swarmpal.io.PalDataTreeAccessor(datatree)[source]#

Provide custom attributes and methods on XArray DataTrees for SwarmPAL functionality.

See e.g. Unidata/MetPy

_datatree#
apply(palprocess: PalProcess) → xarray.DataTree[source]#
quicklook(toolbox: str | None = None)[source]#
property pal_meta: dict#
property magnetic_model_names: list[str]#

List of the model names used in the dataset

property magnetic_model_name: str#

Model name if one and only one has been set

magnetic_residual(model: str = '') → xarray.DataArray[source]#

Magnetic data-model residual in NEC frame

to_cdf(filename: str, leaf: str, handler: str = 'pycdfpp') → None[source]#

Write one leaf of the datatree to a CDF file

Parameters:

  • filename (str) – Name of the file to create

  • leaf (str) – Location within the datatree

  • handler (str) – CDF handler to use. Can be “pycdfpp” or “cdflib”. Defaults to “pycdfpp”.

class swarmpal.io.PalProcess(config: dict | None = None, active_tree: str = '/', inplace: bool = True)[source]#

Bases: abc.ABC

Abstract class to define processes to act on datatrees

_active_tree = '/'#
property process_name: str#
Abstractmethod:

abstractmethod set_config(output_dataset: str = '', **kwargs)[source]#
property output_dataset#
property active_tree: str#

Defines which branch of the datatree will be used

property config: dict#

Dictionary that configures the process behaviour

__call__(datatree) → xarray.DataTree[source]#

Run the process, defined in _call, to update the datatree

abstractmethod _call(datatree) → xarray.DataTree[source]#
swarmpal.io.create_paldata(*paldataitems: PalDataItem, **paldataitems_kw: PalDataItem) → xarray.DataTree[source]#

Generates a Datatree from a number of PalDataItems

Returns:

A Datatree containing Datasets defined from each PalDataItem

Return type:

Datatree

Examples

>>> from swarmpal.io import create_paldata, PalDataItem
>>>
>>> # Parameters to control a particular data request
>>> data_params = dict(
>>>     collection="SW_OPER_MAGA_LR_1B",
>>>     measurements=["B_NEC"],
>>>     models=["IGRF"],
>>>     start_time="2016-01-01T00:00:00",
>>>     end_time="2016-01-01T03:00:00",
>>>     server_url="https://vires.services/ows",
>>>     options=dict(asynchronous=False, show_progress=False),
>>> )
>>> # Create the datatree from a list of items
>>> data = create_paldata(
>>>     PalDataItem.from_vires(**data_params)
>>> )
>>> # Create the datatree from labelled items
>>> data = create_paldata(
>>>     one=PalDataItem.from_vires(**data_params),
>>>     two=PalDataItem.from_vires(**data_params),
>>> )
Contents