Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,11 @@
- Removed redundant `via` access link from the OSC STAC collection; access is already expressed via typed assets (`zarr-data`, `zarr-consolidated-metadata`) on the STAC item.
- `osc_project` is now omitted from `OscDatasetStacGenerator` when not provided, preserving the callee's default instead of passing `None`.

## Changes in 0.2.0 (in Development)
## Changes in 0.2.0 (in development)

- Automatically derives OSC CF parameters from data variable metadata when `cf_params` is not provided.
- Supports temporal coordinates whose names contain `time`.
- The `osc:variables` extension field now contains the variable names used in the cube rather than the `standard_name` values from the data variable attributes.
- Generated project collection now includes required STAC extensions (`osc`, `themes`, `contacts`) and OSC-mandatory fields (`osc:type`, `osc:status`, `themes`, `contacts`) to pass OSC catalog validation.
- Added optional `osc_project_url` field to the dataset config; used as the `via` link in the project collection. Falls back to `documentation_link` if omitted; defaults to the existing DeepESDL project collection when neither is provided.
- `dataset_status` now defaults to `"ongoing"` when not specified in the dataset config.
Expand Down
29 changes: 27 additions & 2 deletions deep_code/tests/utils/test_dataset_stac_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,24 @@ def test_get_temporal_extent(self):
self.assertEqual(interval[0], datetime(2023, 1, 1, 0, 0))
self.assertEqual(interval[1], datetime(2023, 1, 2, 0, 0))

def test_get_temporal_extent_uses_all_time_coordinates(self):
"""Temporal bounds include all coordinates whose names contain time."""
dataset = Dataset(
coords={
"time_start": ("time_start", [np.datetime64("2023-01-03")]),
"time_end": ("time_end", [np.datetime64("2023-01-05")]),
}
)
extent = self.generator._get_temporal_extent(dataset)
self.assertEqual(
extent.intervals[0], [datetime(2023, 1, 3), datetime(2023, 1, 5)]
)

def test_get_cf_params_from_data_variables(self):
"""CF parameters are derived from variable names and attributes."""
params = self.generator._get_cf_params(self.mock_dataset)
self.assertEqual([param["name"] for param in params], ["var1", "var2"])

def test_get_variables(self):
"""Test variable ID extraction."""
vars_ = self.generator.get_variable_ids(self.mock_dataset)
Expand Down Expand Up @@ -1042,7 +1060,11 @@ def setUp(self):
"sst": (
("time", "lat", "lon"),
np.random.rand(2, 3, 3),
{"units": "K", "long_name": "Sea surface temperature"},
{
"units": "K",
"standard_name": "sea_surface_temperature",
"long_name": "Sea surface temperature",
},
),
"chl": (
("time", "lat", "lon"),
Expand Down Expand Up @@ -1193,7 +1215,10 @@ def test_build_prr_collection(self):
self.assertEqual(ef["osc:region"], "Global")
self.assertCountEqual(ef["osc:variables"], ["sst", "chl"])
self.assertEqual(ef["osc:missions"], ["sentinel-3"])
self.assertEqual(ef["cf:parameter"], [{"name": "prr-collection"}])
self.assertEqual(
ef["cf:parameter"],
[{"name": "sea_surface_temperature"}],
)
self.assertIn("processing:datetime", ef)

# Extensions declared.
Expand Down
4 changes: 1 addition & 3 deletions deep_code/tools/new.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ def generate_dataset_template(output_path: str | None = None) -> str:
],
"collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]",
"coord_position": "[OPTIONAL: Position of the coordinates within each grid cell. 'center' assumes coordinates represent cell centers, 'left' assumes they represent the left/bottom edge, and 'right' assumes they represent the right/top edge.]",
"cf_parameter": [
{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}
],
"cf_parameter": [{"name": "[OPTIONAL: CF standard name]"}],
}

# Fields used only by `deep-code generate-prr-collection` to build a
Expand Down
42 changes: 29 additions & 13 deletions deep_code/utils/dataset_stac_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,17 +250,31 @@ def _get_spatial_extent(
@staticmethod
def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent:
"""Extract temporal extent from the dataset."""
dataset = dataset
if "time" in dataset.coords:
time_coords = [name for name in dataset.coords if "time" in name]
if time_coords:
try:
# Convert the time bounds to datetime objects
time_min = pd.to_datetime(dataset.time.min().values).to_pydatetime()
time_max = pd.to_datetime(dataset.time.max().values).to_pydatetime()
time_min = min(
pd.to_datetime(dataset[name].min().values) for name in time_coords
).to_pydatetime()
time_max = max(
pd.to_datetime(dataset[name].max().values) for name in time_coords
).to_pydatetime()
return TemporalExtent([[time_min, time_max]])
except Exception as e:
raise ValueError(f"Failed to parse temporal extent: {e}")
else:
raise ValueError("Dataset does not have a 'time' coordinate.")
raise ValueError("Dataset does not have a '*time*' coordinate.")

@staticmethod
def _get_cf_params(dataset: xr.Dataset) -> list:
params = []

for var_name, data_var in dataset.data_vars.items():
attrs = data_var.attrs
if "standard_name" in attrs:
params.append({"name": attrs["standard_name"]})

return params

@staticmethod
def _normalize_name(name: str | None) -> str | None:
Expand Down Expand Up @@ -357,13 +371,12 @@ def extract_metadata_for_variable(self, variable_data) -> dict:
"""Extract metadata for a single variable."""
long_name = variable_data.attrs.get("long_name")
standard_name = variable_data.attrs.get("standard_name", "unknown")
variable_id = (
variable_data.name if standard_name == "unknown" else standard_name
)
variable_id = variable_data.name
description = variable_data.attrs.get("description", long_name)
gcmd_keyword_url = variable_data.attrs.get("gcmd_keyword_url")
return {
"variable_id": self._normalize_name(variable_id),
"variable_id": variable_id,
"standard_name": standard_name,
"description": description,
"gcmd_keyword_url": gcmd_keyword_url,
}
Expand All @@ -386,7 +399,7 @@ def get_variable_ids(
# Remove 'crs' and 'spatial_ref' from the list if they exist, note that
# spatial_ref will be normalized to spatial-ref in variable_ids and skipped.
return [
var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"]
var_id for var_id in variable_ids if var_id not in ["crs", "spatial_ref"]
]

def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]:
Expand Down Expand Up @@ -1186,7 +1199,10 @@ def build_prr_collection(self) -> Collection:
osc_extension.osc_region = self.osc_region
osc_extension.osc_variables = variables
osc_extension.osc_missions = self.osc_missions
osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}]
if self.cf_params:
osc_extension.cf_parameter = self.cf_params
else:
osc_extension.cf_parameter = self._get_cf_params(dataset_ref)

now_iso = datetime.now(timezone.utc).isoformat()
collection.extra_fields["created"] = now_iso
Expand Down Expand Up @@ -1399,7 +1415,7 @@ def build_dataset_stac_collection(
if self.cf_params:
osc_extension.cf_parameter = self.cf_params
else:
osc_extension.cf_parameter = [{"name": self.collection_id}]
osc_extension.cf_parameter = self._get_cf_params(dataset)

# Add creation and update timestamps for the collection
now_iso = datetime.now(timezone.utc).isoformat()
Expand Down
4 changes: 3 additions & 1 deletion docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ The output conforms to the
[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html)
when the PRR fields are set in the config. If any required field is missing, the command
still runs but logs a warning listing what is needed for full conformance. See
[PRR collection fields](configuration.md#prr-collection-fields).
[PRR collection fields](configuration.md#prr-collection-fields). Before sending the collection JSON to the PRR team,
you can validate the generated collection using the [PRR STAC Collection Editor](https://eoresults.esa.int/prr_collection_editor.html).


Options:

Expand Down
Loading