Skip to content

Schemas

Schema

data_joinery.Schema 

Schema(model)

Define DataFrame fields from a dataclass or Pydantic model.

PARAMETER DESCRIPTION
model

Class whose fields define the schema.

TYPE: type[T]

ATTRIBUTE DESCRIPTION
model

Class used to define the schema.
model_schema

Parsed fields from the model.
Source code in src/data_joinery/schemas/schema.py 
44
45
46
def __init__(self, model: type[T]):
    self.model = model
    self._model_schema = ModelSchema.from_model(model)

model instance-attribute 

model = model

native_schema 

native_schema(schema_type)

Return this schema in a backend's native schema type.

PARAMETER DESCRIPTION
schema_type

Native schema class to create, such as a Spark StructType or Polars Schema.

TYPE: type[NativeSchemaT]

RETURNS DESCRIPTION
NativeSchemaT

The compiled native schema.
Source code in src/data_joinery/schemas/schema.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def native_schema[NativeSchemaT](
    self, schema_type: type[NativeSchemaT]
) -> NativeSchemaT:
    """Return this schema in a backend's native schema type.

    Args:
        schema_type: Native schema class to create, such as a Spark
            ``StructType`` or Polars ``Schema``.

    Returns:
        The compiled native schema.
    """
    backend = _require_backend(
        backend_for_schema_type(schema_type), schema_type, kind="schema"
    )
    native_schema = backend.compile_schema(self._model_schema)
    if not isinstance(native_schema, schema_type):
        raise TypeError(
            f"Backend '{backend.name}' returned schema type "
            f"{type(native_schema).__name__}; expected {schema_type.__name__}. "
            "Fix the backend's compile_schema() result."
        )
    return native_schema

create_dataframe 

create_dataframe(rows, frame_type, **kwargs)

Create a DataFrame from model instances.

PARAMETER DESCRIPTION
rows

Model instances to include in the DataFrame.

TYPE: Sequence[T]

frame_type

DataFrame class to create.

TYPE: type[FrameT]

**kwargs

Extra options passed to the DataFrame backend.

TYPE: object DEFAULT: {}

RETURNS DESCRIPTION
FrameT

A DataFrame of the requested type.
Source code in src/data_joinery/schemas/schema.py 
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def create_dataframe[FrameT](
    self,
    rows: Sequence[T],
    frame_type: type[FrameT],
    **kwargs: object,
) -> FrameT:
    """Create a DataFrame from model instances.

    Args:
        rows: Model instances to include in the DataFrame.
        frame_type: DataFrame class to create.
        **kwargs: Extra options passed to the DataFrame backend.

    Returns:
        A DataFrame of the requested type.
    """
    backend = _require_backend(
        backend_for_frame_type(frame_type), frame_type, kind="dataframe"
    )
    frame = backend.create_dataframe(rows, self._model_schema, frame_type, **kwargs)
    if not isinstance(frame, frame_type):
        raise TypeError(
            f"Backend '{backend.name}' returned DataFrame type "
            f"{type(frame).__name__}; expected {frame_type.__name__}. "
            "Fix the backend's create_dataframe() result."
        )
    return frame

coerce_dataframe 

coerce_dataframe(dataframe, mode='strict')

Validate and adapt a DataFrame to this schema.

strict rejects extra fields and type mismatches. Use project_top_level to remove extra top-level fields, project to remove extra nested fields, or project_cast to also cast field types. Missing fields cause an error in every mode.

PARAMETER DESCRIPTION
dataframe

DataFrame to validate and adapt.

TYPE: FrameT

mode

Schema coercion mode. Defaults to strict.

TYPE: CoercionMode DEFAULT: 'strict'

RETURNS DESCRIPTION
FrameT

A DataFrame of the same backend type.
RAISES DESCRIPTION
SchemaCoercionError

If the DataFrame cannot match the schema.
Source code in src/data_joinery/schemas/schema.py 
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
def coerce_dataframe(
    self, dataframe: FrameT, mode: CoercionMode = "strict"
) -> FrameT:
    """Validate and adapt a DataFrame to this schema.

    ``strict`` rejects extra fields and type mismatches. Use
    ``project_top_level`` to remove extra top-level fields, ``project``
    to remove extra nested fields, or ``project_cast`` to also cast field types.
    Missing fields cause an error in every mode.

    Args:
        dataframe: DataFrame to validate and adapt.
        mode: Schema coercion mode. Defaults to ``strict``.

    Returns:
        A DataFrame of the same backend type.

    Raises:
        SchemaCoercionError: If the DataFrame cannot match the schema.
    """
    backend = _require_backend(
        backend_for_value(dataframe), type(dataframe), kind="dataframe"
    )
    try:
        frame = backend.coerce_dataframe(dataframe, self._model_schema, mode)
    except SchemaCoercionError as error:
        raise SchemaCoercionError(
            error.mode,
            error.violations,
            location=f"Schema '{self.model.__name__}'",
        ) from None
    if not isinstance(frame, type(dataframe)):
        raise TypeError(
            f"Backend '{backend.name}' returned DataFrame type "
            f"{type(frame).__name__}; expected {type(dataframe).__name__}. "
            "Fix the backend's coerce_dataframe() result."
        )
    return cast(FrameT, frame)