From da9ca2399c28b5ce31823afb4417395788012eb5 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Mon, 4 Dec 2023 15:42:04 +1100 Subject: [PATCH 1/4] MAINT: get orsopy working in 3.12 --- .github/workflows/pytest.yml | 2 +- orsopy/fileio/__init__.py | 14 ++- orsopy/fileio/base.py | 166 ++++++++++++++++++++++-------- orsopy/fileio/data_source.py | 112 ++++++++++++++++++-- orsopy/fileio/model_language.py | 176 ++++++++++++++++++++++++++++---- orsopy/fileio/orso.py | 7 +- orsopy/fileio/reduction.py | 50 ++++++++- 7 files changed, 449 insertions(+), 78 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3af8c863..ec36e05f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, '3.10', '3.11'] + python-version: [3.8, 3.9, '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 diff --git a/orsopy/fileio/__init__.py b/orsopy/fileio/__init__.py index 129821e1..a42a9056 100644 --- a/orsopy/fileio/__init__.py +++ b/orsopy/fileio/__init__.py @@ -2,10 +2,22 @@ Implementation of the Orso class that defined the header. """ +import sys + from .base import (Column, ComplexValue, ErrorColumn, File, Header, Person, Value, ValueRange, ValueVector, - _read_header_data, _validate_header_data) + _read_header_data, _validate_header_data, ORSO_DATACLASSES) from .data_source import DataSource, Experiment, InstrumentSettings, Measurement, Polarization, Sample from .orso import ORSO_VERSION, Orso, OrsoDataset, load_orso, save_orso, load_nexus, save_nexus from .reduction import Reduction, Software +this_module = sys.modules[__name__] + +for _o in dir(this_module): + cls = getattr(this_module, _o) + try: + if issubclass(cls, Header): + ORSO_DATACLASSES[_o] = cls + except Exception: + pass + __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/orsopy/fileio/base.py b/orsopy/fileio/base.py index 7ae3545c..0224896d 100644 --- a/orsopy/fileio/base.py +++ b/orsopy/fileio/base.py @@ -88,40 +88,6 @@ def _custom_init_fn(fieldsarg, frozen, has_post_init, self_name, globals): ORSO_DATACLASSES = dict() -def orsodataclass(cls: type): - ORSO_DATACLASSES[cls.__name__] = cls - attrs = cls.__dict__ - bases = cls.__bases__ - if "__annotations__" in attrs and len([k for k in attrs["__annotations__"].keys() if not k.startswith("_")]) > 0: - # only applies to dataclass children of Header - # add optional comment attribute, needs to come last - attrs["__annotations__"]["comment"] = Optional[str] - setattr(cls, "comment", field(default=None)) - - # create the _orso_optional attribute - orso_optionals = [] - for fname, ftype in attrs["__annotations__"].items(): - if type(None) in get_args(ftype): - orso_optionals.append(fname) - for base in bases: - if hasattr(base, "_orso_optionals"): - orso_optionals += getattr(base, "_orso_optionals") - setattr(cls, "_orso_optionals", orso_optionals) - out = dataclass(cls, repr=False, init=False) - fieldsarg = getattr(out, _FIELDS) - - # Generate custom __init__ method that allows arbitrary extra keyword arguments - has_post_init = hasattr(out, _POST_INIT_NAME) - # Include InitVars and regular fields (so, not ClassVars). - flds = [f for f in fieldsarg.values() if f._field_type in (_FIELD, _FIELD_INITVAR)] - init_fun = _custom_init_fn(flds, False, has_post_init, "self", globals()) - _set_new_attribute(out, "__init__", init_fun) - - return out - else: - return cls - - class ORSOResolveError(ValueError): pass @@ -133,7 +99,14 @@ class Header: _orso_optionals: List[str] = [] + def __init__(self): + self._orso_optionals = [] + def __post_init__(self): + for fname, ftype in self.__annotations__.items(): + if type(None) in get_args(ftype): + self._orso_optionals.append(fname) + """Make sure Header types are correct.""" for fld in fields(self): attr = getattr(self, fld.name, None) @@ -510,7 +483,7 @@ def represent_data(self, data): unit_registry = None -@orsodataclass +@dataclass class ErrorValue(Header): """ Information about errors on a value. @@ -520,9 +493,21 @@ class ErrorValue(Header): error_type: Optional[Literal["uncertainty", "resolution"]] = None value_is: Optional[Literal["sigma", "FWHM"]] = None distribution: Optional[Literal["gaussian", "triangular", "uniform", "lorentzian"]] = None + comment: Optional[str] = None yaml_representer = Header.yaml_representer_compact + def __init__(self, error_value, error_type=None, value_is=None, distribution=None, *, comment=None, **kwds): + super(ErrorValue, self).__init__() + self.error_value = error_value + self.error_type = error_type + self.value_is = value_is + self.distribution = distribution + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + @property def sigma(self): """ @@ -560,7 +545,7 @@ def sigma(self): return self.error_value -@orsodataclass +@dataclass class Value(Header): """ A value or list of values with an optional unit. @@ -569,9 +554,20 @@ class Value(Header): magnitude: float unit: Optional[str] = field(default=None, metadata={"description": "SI unit string"}) error: Optional[ErrorValue] = None + comment: Optional[str] = None yaml_representer = Header.yaml_representer_compact + def __init__(self, magnitude, unit=None, error=None, *, comment=None, **kwds): + super(Value, self).__init__() + self.magnitude = magnitude + self.unit = unit + self.error = error + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def __repr__(self): """ Make representation more readability by removing names of default arguments. @@ -598,7 +594,7 @@ def as_unit(self, output_unit): return val.to(output_unit).magnitude -@orsodataclass +@dataclass class ComplexValue(Header): """ A value or list of values with an optional unit. @@ -608,9 +604,21 @@ class ComplexValue(Header): imag: Optional[float] = None unit: Optional[str] = field(default=None, metadata={"description": "SI unit string"}) error: Optional[ErrorValue] = None + comment: Optional[str] = None yaml_representer = Header.yaml_representer_compact + def __init__(self, real, imag=None, unit=None, error=None, *, comment=None, **kwds): + super(ComplexValue, self).__init__() + self.real = real + self.imag = imag + self.unit = unit + self.error = error + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def __repr__(self): """ Make representation more readability by removing names of default arguments. @@ -642,7 +650,7 @@ def as_unit(self, output_unit): return val.to(output_unit).magnitude -@orsodataclass +@dataclass class ValueRange(Header): """ A range or list of ranges with mins, maxs, and an optional unit. @@ -651,9 +659,20 @@ class ValueRange(Header): min: float max: float unit: Optional[str] = field(default=None, metadata={"description": "SI unit string"}) + comment: Optional[str] = None yaml_representer = Header.yaml_representer_compact + def __init__(self, min, max, unit=None, *, comment=None, **kwds): + super(ValueRange, self).__init__() + self.min = min + self.max = max + self.unit = unit + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def as_unit(self, output_unit): """ Returns a (min, max) tuple of values as converted to the given unit. @@ -672,7 +691,7 @@ def as_unit(self, output_unit): return (vmin.to(output_unit).magnitude, vmax.to(output_unit).magnitude) -@orsodataclass +@dataclass class ValueVector(Header): """ A vector or list of vectors with an optional unit. @@ -692,9 +711,22 @@ class ValueVector(Header): z: float unit: Optional[str] = field(default=None, metadata={"description": "SI unit string"}) error: Optional[ErrorValue] = None + comment: Optional[str] = None yaml_representer = Header.yaml_representer_compact + def __init__(self, x, y, z, unit=None, error=None, *, comment=None, **kwds): + super(ValueVector, self).__init__() + self.x = x + self.y = y + self.z = z + self.unit = unit + self.error = error + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def as_unit(self, output_unit): """ Returns a (x, y, z) tuple of values as converted to the given unit. @@ -714,7 +746,7 @@ def as_unit(self, output_unit): return (vx.to(output_unit).magnitude, vy.to(output_unit).magnitude, vz.to(output_unit).magnitude) -@orsodataclass +@dataclass class Person(Header): """ Information about a person, including name, affiliation(s), and contact @@ -725,8 +757,20 @@ class Person(Header): affiliation: str contact: Optional[str] = field(default=None, metadata={"description": "Contact (email) address"}) + comment: Optional[str] = None -@orsodataclass + def __init__(self, name, affiliation, contact=None, *, comment=None, **kwds): + super(Person, self).__init__() + self.name = name + self.affiliation = affiliation + self.contact = contact + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + + +@dataclass class Column(Header): """ Information about a data column. @@ -738,10 +782,22 @@ class Column(Header): default=None, metadata={"physical_quantity": "A description of the column"} ) + comment: Optional[str] = None + yaml_representer = Header.yaml_representer_compact + def __init__(self, name, unit=None, physical_quantity=None, *, comment=None, **kwds): + super(Column, self).__init__() + self.name = name + self.unit = unit + self.physical_quantity = physical_quantity + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() -@orsodataclass + +@dataclass class ErrorColumn(Header): """ Information about a data column. @@ -752,8 +808,21 @@ class ErrorColumn(Header): value_is: Optional[Literal["sigma", "FWHM"]] = None distribution: Optional[Literal["gaussian", "triangular", "uniform", "lorentzian"]] = None + comment: Optional[str] = None + yaml_representer = Header.yaml_representer_compact + def __init__(self, error_of, error_type=error_type, value_is=None, distribution=None, *, comment=None, **kwds): + super(ErrorColumn, self).__init__() + self.error_of = error_of + self.error_type = error_type + self.value_is = value_is + self.distribution = distribution + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + @property def name(self): """ @@ -796,7 +865,7 @@ def to_sigma(self): return 1.0 -@orsodataclass +@dataclass class File(Header): """ A file with file path and a last modified timestamp. @@ -812,6 +881,17 @@ class File(Header): }, ) + comment: Optional[str] = None + + def __init__(self, file, timestamp=None, *, comment=None, **kwds): + super(File, self).__init__() + self.file = file + self.timestamp = timestamp + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def __post_init__(self): """ Assigns a timestamp for file creation if not defined. diff --git a/orsopy/fileio/data_source.py b/orsopy/fileio/data_source.py index 592f6334..ea37651c 100644 --- a/orsopy/fileio/data_source.py +++ b/orsopy/fileio/data_source.py @@ -1,14 +1,14 @@ """ Implementation of the data_source for the ORSO header. """ -from dataclasses import field +from dataclasses import field, dataclass from datetime import datetime from enum import Enum from typing import Dict, List, Optional, Union import yaml -from .base import ComplexValue, File, Header, Person, Value, ValueRange, ValueVector, orsodataclass +from .base import ComplexValue, File, Header, Person, Value, ValueRange, ValueVector from .model_language import SampleModel # typing stuff introduced in python 3.8 @@ -18,7 +18,7 @@ from .typing_backport import Literal -@orsodataclass +@dataclass class Experiment(Header): """ A definition of the experiment performed. @@ -42,8 +42,36 @@ class Experiment(Header): proposalID: Optional[str] = None doi: Optional[str] = None - -@orsodataclass + comment: Optional[str] = None + + def __init__( + self, + title, + instrument, + start_date, + probe, + facility=None, + proposalID=None, + doi=None, + *, + comment=None, + **kwds + ): + super(Experiment, self).__init__() + self.title = title + self.instrument = instrument + self.start_date = start_date + self.probe = probe + self.facility = facility + self.proposalID = proposalID + self.doi = doi + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + + +@dataclass class Sample(Header): """ A description of the sample measured. @@ -75,6 +103,35 @@ class Sample(Header): ) model: Optional[SampleModel] = None + comment: Optional[str] = None + + def __init__( + self, + name, + category=None, + composition=None, + description=None, + size=None, environment=None, + sample_parameters=None, + model=None, + *, + comment=None, + **kwds + ): + super(Sample, self).__init__() + self.name = name + self.category = category + self.composition = composition + self.description = description + self.size = size + self.environment = environment + self.sample_parameters = sample_parameters + self.model = model + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + class Polarization(str, Enum): """ @@ -118,7 +175,7 @@ def yaml_representer(self, dumper: yaml.Dumper): return dumper.represent_str(output) -@orsodataclass +@dataclass class InstrumentSettings(Header): """ Settings associated with the instrumentation. @@ -147,8 +204,21 @@ class InstrumentSettings(Header): __repr__ = Header._staggered_repr + comment: Optional[str] = None + + def __init__(self, incident_angle, wavelength, polarization=None, configuration=None, *, comment=None, **kwds): + super(InstrumentSettings, self).__init__() + self.incident_angle = incident_angle + self.wavelength = wavelength + self.polarization = polarization + self.configuration = configuration + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + -@orsodataclass +@dataclass class Measurement(Header): """ The measurement elements for the header. @@ -167,8 +237,21 @@ class Measurement(Header): __repr__ = Header._staggered_repr + comment: Optional[str] = None -@orsodataclass + def __init__(self, instrument_settings, data_files, additional_files=None, scheme=None, *, comment=None, **kwds): + super(Measurement, self).__init__() + self.instrument_settings = instrument_settings + self.data_files = data_files + self.additional_files = additional_files + self.scheme = scheme + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + + +@dataclass class DataSource(Header): """ The data_source object definition. @@ -188,3 +271,16 @@ class DataSource(Header): _orso_optionals = [] __repr__ = Header._staggered_repr + + comment: Optional[str] = None + + def __init__(self, owner, experiment, sample, measurement, *, comment=None, **kwds): + super(DataSource, self).__init__() + self.owner = owner + self.experiment = experiment + self.sample = sample + self.measurement = measurement + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() diff --git a/orsopy/fileio/model_language.py b/orsopy/fileio/model_language.py index dca1d827..02b253ea 100644 --- a/orsopy/fileio/model_language.py +++ b/orsopy/fileio/model_language.py @@ -8,10 +8,10 @@ from typing import Any, Dict, List, Optional, Union -from ..dataclasses import field +from ..dataclasses import field, dataclass from ..utils.chemical_formula import Formula from ..utils.density_resolver import DensityResolver -from .base import ComplexValue, Header, Value, orsodataclass +from .base import ComplexValue, Header, Value DENSITY_RESOLVERS: List[DensityResolver] = [] @@ -25,7 +25,7 @@ def find_idx(string, start, value): return next_idx -@orsodataclass +@dataclass class ModelParameters(Header): roughness: Value = field(default_factory=lambda: Value(0.3, "nm")) length_unit: str = "nm" @@ -34,8 +34,34 @@ class ModelParameters(Header): sld_unit: str = "1/angstrom^2" magnetic_moment_unit: str = "muB" - -@orsodataclass + comment: Optional[str] = None + + def __init__( + self, + roughness=Value(0.3, unit="nm"), + length_unit="nm", + mass_density_unit="g/cm^3", + number_density_unit="1/nm^3", + sld_unit="1/angstrom^2", + magnetic_moment_unit="muB", + *, + comment=None, + **kwds + ): + super(ModelParameters, self).__init__() + self.roughness = roughness + self.length_unit = length_unit + self.mass_density_unit = mass_density_unit + self.number_density_unit = number_density_unit + self.sld_unit = sld_unit + self.magnetic_moment_unit = magnetic_moment_unit + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + + +@dataclass class Material(Header): formula: Optional[str] = None mass_density: Optional[Union[float, Value]] = None @@ -43,11 +69,35 @@ class Material(Header): sld: Optional[Union[float, ComplexValue, Value]] = None magnetic_moment: Optional[Union[float, Value]] = None relative_density: Optional[float] = None - - original_name = None - - def __post_init__(self): - super().__post_init__() + original_name: Optional[Any] = None + + comment: Optional[str] = None + + def __init__( + self, + formula=None, + mass_density=None, + number_density=None, + sld=None, + magnetic_moment=None, + relative_density=None, + original_name=None, + *, + comment=None, + **kwds + ): + super(Material, self).__init__() + self.formula = formula + self.mass_density = mass_density + self.number_density = number_density + self.sld = sld + self.magnetic_moment = magnetic_moment + self.relative_density = relative_density + self.original_name = original_name + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() def resolve_defaults(self, defaults: ModelParameters): if self.formula is None and self.sld is None: @@ -149,11 +199,22 @@ def get_sld(self, xray_energy=None) -> complex: return 0.0j -@orsodataclass +@dataclass class Composit(Header): composition: Dict[str, float] - original_name = None + original_name: Optional[Any] = None + + comment: Optional[str] = None + + def __init__(self, composition, original_name=None, *, comment=None, **kwds): + super(Composit, self).__init__() + self.composition = composition + self.original_name = original_name + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() def resolve_names(self, resolvable_items): self._composition_materials = {} @@ -199,7 +260,7 @@ def get_sld(self, xray_energy=None): CACHED_MATERIALS = {} -@orsodataclass +@dataclass class Layer(Header): thickness: Optional[Union[float, Value]] = None roughness: Optional[Union[float, Value]] = None @@ -208,8 +269,29 @@ class Layer(Header): original_name = None - def __post_init__(self): - super().__post_init__() + comment: Optional[str] = None + + def __init__( + self, + thickness=None, + roughness=None, + material=None, + composition=None, + original_name=None, + *, + comment=None, + **kwds + ): + super(Layer, self).__init__() + self.thickness = thickness + self.roughness = roughness + self.material = material + self.composition = composition + self.original_name = original_name + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() def resolve_names(self, resolvable_items): if self.material is None and self.composition is None and self.original_name is None: @@ -282,7 +364,7 @@ def generate_material(self): ) -@orsodataclass +@dataclass class SubStack(Header): repetitions: int = 1 stack: Optional[str] = None @@ -291,7 +373,35 @@ class SubStack(Header): arguments: Optional[List[Any]] = None keywords: Optional[Dict[str, Any]] = None - original_name = None + original_name: Optional[Any] = None + + comment: Optional[str] = None + + def __init__( + self, + repetitions=1, + stack=None, + sequence=None, + represents=None, + arguments=None, + keywords=None, + original_name=None, + *, + comment=None, + **kwds + ): + super(SubStack, self).__init__() + self.repetitions = repetitions + self.stack = stack + self.sequence = sequence + self.represents = represents + self.arguments = arguments + self.keywords = keywords + self.original_name = original_name + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() def resolve_names(self, resolvable_items): if self.stack is None and self.sequence is None: @@ -355,7 +465,7 @@ def resolve_to_layers(self): return layers * self.repetitions -@orsodataclass +@dataclass class SampleModel(Header): stack: str origin: Optional[str] = None @@ -366,6 +476,36 @@ class SampleModel(Header): globals: Optional[ModelParameters] = None reference: Optional[str] = None + comment: Optional[str] = None + + def __init__( + self, + stack, + origin=None, + sub_stacks=None, + layers=None, + materials=None, + composits=None, + globals=None, + reference=None, + *, + comment=None, + **kwds + ): + super(SampleModel, self).__init__() + self.stack = stack + self.origin = origin + self.sub_stacks = sub_stacks + self.layers = layers + self.materials = materials + self.composits = composits + self.globals = globals + self.reference = reference + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + def __post_init__(self): super().__post_init__() names = [] diff --git a/orsopy/fileio/orso.py b/orsopy/fileio/orso.py index 8e9983c7..2a6943ad 100644 --- a/orsopy/fileio/orso.py +++ b/orsopy/fileio/orso.py @@ -8,8 +8,8 @@ import numpy as np import yaml -from .base import (JSON_MIMETYPE, ORSO_DATACLASSES, Column, ErrorColumn, Header, _dict_diff, _nested_update, - _possibly_open_file, _read_header_data, orsodataclass) +from .base import (ORSO_DATACLASSES, JSON_MIMETYPE, Column, ErrorColumn, Header, _dict_diff, _nested_update, + _possibly_open_file, _read_header_data) from .data_source import DataSource from .reduction import Reduction @@ -19,7 +19,7 @@ ) -@orsodataclass +@dataclass class Orso(Header): """ The Orso object collects the necessary metadata. @@ -50,6 +50,7 @@ def __init__( data_set: Optional[Union[int, str]] = None, **user_data, ): + super(Orso, self).__init__() self.data_source = data_source self.reduction = reduction self.columns = columns diff --git a/orsopy/fileio/reduction.py b/orsopy/fileio/reduction.py index 28693f11..2697699c 100644 --- a/orsopy/fileio/reduction.py +++ b/orsopy/fileio/reduction.py @@ -4,13 +4,13 @@ import datetime -from dataclasses import field +from dataclasses import field, dataclass from typing import List, Optional, Union -from .base import Header, Person, orsodataclass +from .base import Header, Person -@orsodataclass +@dataclass class Software(Header): """ Software description. @@ -24,10 +24,22 @@ class Software(Header): version: Optional[str] = None platform: Optional[str] = None + comment: Optional[str] = None + yaml_representer = Header.yaml_representer_compact + def __init__(self, name, version=None, platform=None, *, comment=None, **kwds): + super(Software, self).__init__() + self.name = name + self.version = version + self.platform = platform + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() + -@orsodataclass +@dataclass class Reduction(Header): """ A description of the reduction that has been performed. @@ -54,3 +66,33 @@ class Reduction(Header): binary: Optional[str] = field(default=None, metadata={"description": "Path to full information file"}) __repr__ = Header._staggered_repr + + comment: Optional[str] = None + + def __init__( + self, + software, + timestamp=None, + creator=None, + corrections=None, + computer=None, + call=None, + script=None, + binary=None, + *, + comment=None, + **kwds + ): + super(Reduction, self).__init__() + self.software = software + self.timestamp = timestamp + self.creator = creator + self.corrections = corrections + self.computer = computer + self.call = call + self.script = script + self.binary = binary + for k, v in kwds.items(): + setattr(self, k, v) + self.comment = comment + self.__post_init__() From 2fc509ee22e9272d8b9da0a6eb1bacbdd3391cba Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Mon, 4 Dec 2023 22:18:36 +1100 Subject: [PATCH 2/4] MAINT: fix schema --- orsopy/_dataclasses.py | 2 + orsopy/dataclasses.py | 22 --------- orsopy/fileio/base.py | 43 +---------------- orsopy/fileio/model_language.py | 2 +- orsopy/fileio/orso.py | 5 +- orsopy/fileio/schema/refl_header.schema.json | 18 ++----- orsopy/fileio/schema/refl_header.schema.yaml | 21 +++++---- tools/header_schema.py | 49 +++++++++++--------- 8 files changed, 51 insertions(+), 111 deletions(-) create mode 100644 orsopy/_dataclasses.py delete mode 100644 orsopy/dataclasses.py diff --git a/orsopy/_dataclasses.py b/orsopy/_dataclasses.py new file mode 100644 index 00000000..fb55e876 --- /dev/null +++ b/orsopy/_dataclasses.py @@ -0,0 +1,2 @@ + +from dataclasses import dataclass, field, fields diff --git a/orsopy/dataclasses.py b/orsopy/dataclasses.py deleted file mode 100644 index a66bc5ea..00000000 --- a/orsopy/dataclasses.py +++ /dev/null @@ -1,22 +0,0 @@ -import sys - -from dataclasses import (_FIELD, _FIELD_INITVAR, _FIELDS, _HAS_DEFAULT_FACTORY, _POST_INIT_NAME, MISSING, _create_fn, - _field_init, _init_param, _set_new_attribute, dataclass, field, fields) - -# change of signature introduced in python 3.10.1 -if sys.version_info >= (3, 10, 1): - _field_init_real = _field_init - - def _field_init(f, frozen, locals, self_name): - return _field_init_real(f, frozen, locals, self_name, False) - - -elif sys.version_info < (3, 7, 0): - # fix bug in python 3.6 when using default_factory for dataclass objects - _orig_field = field - - def field(*args, **opts): - if "default_factory" in opts and not opts["default_factory"] in [list, tuple]: - return opts["default_factory"]() - else: - return _orig_field(*args, **opts) diff --git a/orsopy/fileio/base.py b/orsopy/fileio/base.py index 0224896d..72915284 100644 --- a/orsopy/fileio/base.py +++ b/orsopy/fileio/base.py @@ -16,6 +16,7 @@ from enum import Enum from inspect import isclass from typing import Any, Dict, Generator, List, Optional, TextIO, Tuple, Union +from dataclasses import dataclass, field, fields import numpy as np import yaml @@ -27,9 +28,6 @@ except ImportError: from .typing_backport import Literal, get_args, get_origin -from ..dataclasses import (_FIELD, _FIELD_INITVAR, _FIELDS, _HAS_DEFAULT_FACTORY, _POST_INIT_NAME, MISSING, _create_fn, - _field_init, _init_param, _set_new_attribute, dataclass, field, fields) - def _noop(self, *args, **kw): pass @@ -45,45 +43,6 @@ def _noop(self, *args, **kw): ] = yaml.constructor.SafeConstructor.yaml_constructors["tag:yaml.org,2002:str"] -def _custom_init_fn(fieldsarg, frozen, has_post_init, self_name, globals): - """ - _init_fn from dataclasses adapted to accept additional keywords. - See dataclasses source for comments on code. - """ - seen_default = False - for f in fieldsarg: - if f.init: - if not (f.default is MISSING and f.default_factory is MISSING): - seen_default = True - elif seen_default: - raise TypeError(f"non-default argument {f.name!r} " "follows default argument") - - locals = {f"_type_{f.name}": f.type for f in fieldsarg} - locals.update({"MISSING": MISSING, "_HAS_DEFAULT_FACTORY": _HAS_DEFAULT_FACTORY}) - - body_lines = [] - for f in fieldsarg: - line = _field_init(f, frozen, locals, self_name) - if line: - body_lines.append(line) - - if has_post_init: - params_str = ",".join(f.name for f in fieldsarg if f._field_type is _FIELD_INITVAR) - body_lines.append(f"{self_name}.{_POST_INIT_NAME}({params_str})") - - # processing of additional user keyword arguments - body_lines += ["for k,v in user_kwds.items():", " setattr(self, k, v)"] - - return _create_fn( - "__init__", - [self_name] + [_init_param(f) for f in fieldsarg if f.init] + ["**user_kwds"], - body_lines, - locals=locals, - globals=globals, - return_type=None, - ) - - # register all ORSO classes here: ORSO_DATACLASSES = dict() diff --git a/orsopy/fileio/model_language.py b/orsopy/fileio/model_language.py index 02b253ea..dcc513ae 100644 --- a/orsopy/fileio/model_language.py +++ b/orsopy/fileio/model_language.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional, Union -from ..dataclasses import field, dataclass +from dataclasses import field, dataclass from ..utils.chemical_formula import Formula from ..utils.density_resolver import DensityResolver from .base import ComplexValue, Header, Value diff --git a/orsopy/fileio/orso.py b/orsopy/fileio/orso.py index 2a6943ad..e9f8467b 100644 --- a/orsopy/fileio/orso.py +++ b/orsopy/fileio/orso.py @@ -2,10 +2,11 @@ Implementation of the top level class for the ORSO header. """ -from dataclasses import dataclass, fields +from .._dataclasses import dataclass, fields from typing import BinaryIO, List, Optional, Sequence, TextIO, Union import numpy as np +from numpy.typing import NDArray import yaml from .base import (ORSO_DATACLASSES, JSON_MIMETYPE, Column, ErrorColumn, Header, _dict_diff, _nested_update, @@ -164,7 +165,7 @@ class OrsoDataset: """ info: Orso - data: Union[np.ndarray, Sequence[np.ndarray], Sequence[Sequence]] + data: Union[NDArray, Sequence[NDArray], Sequence[Sequence]] def __post_init__(self): if self.data.shape[1] != len(self.info.columns): diff --git a/orsopy/fileio/schema/refl_header.schema.json b/orsopy/fileio/schema/refl_header.schema.json index e3700221..45c78fdd 100644 --- a/orsopy/fileio/schema/refl_header.schema.json +++ b/orsopy/fileio/schema/refl_header.schema.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/reflectivity/orsopy/v1.0/orsopy/fileio/schema/refl_header.schema.json", "title": "Orso", + "description": "The Orso object collects the necessary metadata.\n\n:param data_source: Information about the origin and ownership of\n the raw data.\n:param reduction: Details of the data reduction that has been\n performed. The content of this section should contain enough\n information to rerun the reduction.\n:param columns: Information about the columns of data that will\n be contained in the file.\n:param data_set: An identifier for the data set, i.e. if there is\n more than one data set in the object.", "type": "object", "properties": { "data_source": { @@ -53,12 +54,6 @@ "type": "null" } ] - }, - "comment": { - "type": [ - "string", - "null" - ] } }, "required": [ @@ -455,6 +450,7 @@ "null" ] }, + "original_name": {}, "comment": { "type": [ "string", @@ -476,6 +472,7 @@ "null" ] }, + "original_name": {}, "comment": { "type": [ "string", @@ -595,6 +592,7 @@ "null" ] }, + "original_name": {}, "comment": { "type": [ "string", @@ -608,13 +606,7 @@ "type": "object", "properties": { "roughness": { - "default": { - "magnitude": 0.3, - "unit": "nm", - "error": null, - "comment": null - }, - "allOf": [ + "anyOf": [ { "$ref": "#/definitions/Value" } diff --git a/orsopy/fileio/schema/refl_header.schema.yaml b/orsopy/fileio/schema/refl_header.schema.yaml index afb82162..8297205d 100644 --- a/orsopy/fileio/schema/refl_header.schema.yaml +++ b/orsopy/fileio/schema/refl_header.schema.yaml @@ -63,6 +63,7 @@ definitions: type: - object - 'null' + original_name: {} required: - composition title: Composit @@ -327,6 +328,7 @@ definitions: - type: number - $ref: '#/definitions/Value' - type: 'null' + original_name: {} relative_density: type: - number @@ -405,13 +407,8 @@ definitions: - string - 'null' roughness: - allOf: + anyOf: - $ref: '#/definitions/Value' - default: - comment: null - error: null - magnitude: 0.3 - unit: nm sld_unit: default: 1/angstrom^2 type: @@ -718,6 +715,7 @@ definitions: type: - object - 'null' + original_name: {} repetitions: default: 1 type: @@ -851,6 +849,13 @@ definitions: - error_of title: sR type: object +description: "The Orso object collects the necessary metadata.\n\n:param data_source:\ + \ Information about the origin and ownership of\n the raw data.\n:param reduction:\ + \ Details of the data reduction that has been\n performed. The content of this\ + \ section should contain enough\n information to rerun the reduction.\n:param\ + \ columns: Information about the columns of data that will\n be contained in\ + \ the file.\n:param data_set: An identifier for the data set, i.e. if there is\n\ + \ more than one data set in the object." properties: columns: additionalItems: @@ -863,10 +868,6 @@ properties: type: - array - 'null' - comment: - type: - - string - - 'null' data_set: anyOf: - type: integer diff --git a/tools/header_schema.py b/tools/header_schema.py index 0771e69c..181c28af 100644 --- a/tools/header_schema.py +++ b/tools/header_schema.py @@ -2,6 +2,13 @@ Generates the schema for an ORSO file, based on the Orso class (from orsopy.fileio.orso) """ + +""" +** NOTE ** +This script is extremely sensitive to the version of pydantic that is installed. +It works with pydantic-1.10.13 and lower, but not with pydantic >= 2.0 +""" + import functools import os @@ -9,31 +16,32 @@ from typing import Any, Dict, List from pydantic.dataclasses import dataclass as _dataclass +from pydantic import ConfigDict + +def schema_extra(schema: Dict[str, Any]) -> None: + for prop, value in schema.get("properties", {}).items(): + value.pop("title", None) -class PydanticConfig: - """ for schema generation, otherwise unused """ + # make the schema accept None as a value for any of the + # Header class attributes. + if "enum" in value: + value["enum"].append(None) - @staticmethod - def schema_extra(schema: Dict[str, Any]) -> None: - for prop, value in schema.get("properties", {}).items(): - value.pop("title", None) + if "type" in value: + value["type"] = [value.pop("type"), "null"] + elif "anyOf" in value: + value["anyOf"].append({"type": "null"}) + # only one $ref e.g. from other model + elif "$ref" in value: + value["anyOf"] = [{"$ref": value.pop("$ref")}] - # make the schema accept None as a value for any of the - # Header class attributes. - if "enum" in value: - value["enum"].append(None) - if "type" in value: - value["type"] = [value.pop("type"), "null"] - elif "anyOf" in value: - value["anyOf"].append({"type": "null"}) - # only one $ref e.g. from other model - elif "$ref" in value: - value["anyOf"] = [{"$ref": value.pop("$ref")}] +# pydantic doesn't like dealing with np.ndarrays +config = ConfigDict(arbitrary_types_allowed=True, schema_extra=schema_extra) -pydantic_dataclass = functools.partial(_dataclass, config=PydanticConfig) +pydantic_dataclass = functools.partial(_dataclass, config=config) ADD_COLUMN_ORDER = True @@ -98,9 +106,8 @@ def add_column_ordering(schema: Dict, column_order: List[str] = COLUMN_ORDER): def main(): # replace the dataclass function in the local import: - from orsopy import dataclasses - - dataclasses.dataclass = pydantic_dataclass + from orsopy import _dataclasses + _dataclasses.dataclass = pydantic_dataclass import orsopy From f38d399ec4fdad9c559bcf8acd2abb4f14997371 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Tue, 5 Dec 2023 12:53:38 +1100 Subject: [PATCH 3/4] MAINT: np.typing not available for cp36 --- orsopy/fileio/orso.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/orsopy/fileio/orso.py b/orsopy/fileio/orso.py index e9f8467b..fdf2979f 100644 --- a/orsopy/fileio/orso.py +++ b/orsopy/fileio/orso.py @@ -6,7 +6,6 @@ from typing import BinaryIO, List, Optional, Sequence, TextIO, Union import numpy as np -from numpy.typing import NDArray import yaml from .base import (ORSO_DATACLASSES, JSON_MIMETYPE, Column, ErrorColumn, Header, _dict_diff, _nested_update, @@ -165,7 +164,7 @@ class OrsoDataset: """ info: Orso - data: Union[NDArray, Sequence[NDArray], Sequence[Sequence]] + data: Union[np.ndrray, Sequence[np.ndarray], Sequence[Sequence]] def __post_init__(self): if self.data.shape[1] != len(self.info.columns): From b199e4493c8e62e6f433cfa21ba0ba931e47762a Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Wed, 6 Dec 2023 09:17:44 +1100 Subject: [PATCH 4/4] BUG: typo --- orsopy/fileio/orso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orsopy/fileio/orso.py b/orsopy/fileio/orso.py index fdf2979f..45d3b7e4 100644 --- a/orsopy/fileio/orso.py +++ b/orsopy/fileio/orso.py @@ -164,7 +164,7 @@ class OrsoDataset: """ info: Orso - data: Union[np.ndrray, Sequence[np.ndarray], Sequence[Sequence]] + data: Union[np.ndarray, Sequence[np.ndarray], Sequence[Sequence]] def __post_init__(self): if self.data.shape[1] != len(self.info.columns):