Skip to content

DataChain

The DataChain class creates a data chain, which is a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data. See DataChain for examples of how to create a chain.

C module-attribute 

C = Column

Column 

Column(
    text, type_=None, is_literal=False, _selectable=None
)

Bases: ColumnClause

Source code in datachain/query/schema.py 
48
49
50
51
52
53
def __init__(self, text, type_=None, is_literal=False, _selectable=None):
    """Dataset column."""
    self.name = ColumnMeta.to_db_name(text)
    super().__init__(
        self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
    )

glob 

glob(glob_str)

Search for matches using glob pattern matching.

Source code in datachain/query/schema.py 
58
59
60
def glob(self, glob_str):
    """Search for matches using glob pattern matching."""
    return self.op("GLOB")(glob_str)

regexp 

regexp(regexp_str)

Search for matches using regexp pattern matching.

Source code in datachain/query/schema.py 
62
63
64
def regexp(self, regexp_str):
    """Search for matches using regexp pattern matching."""
    return self.op("REGEXP")(regexp_str)

read_csv 

read_csv(
    path: (
        str
        | PathLike[str]
        | list[str]
        | list[PathLike[str]]
    ),
    delimiter: str | None = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: int | None = None,
    session: Session | None = None,
    settings: dict | None = None,
    column_types: dict[str, str | DataType] | None = None,
    parse_options: (
        dict[str, str | bool | Callable] | None
    ) = None,
    **kwargs
) -> DataChain

Generate chain from csv files.

Parameters:

  • path (str | PathLike[str] | list[str] | list[PathLike[str]]) –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • delimiter (str | None, default: None ) –

    Character for delimiting columns. Takes precedence if also specified in parse_options. Defaults to ",".

  • header (bool, default: True ) –

    Whether the files include a header row.

  • output (OutputType, default: None ) –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column (str, default: '' ) –

    Created column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • nrows (int | None, default: None ) –

    Optional row limit.

  • session (Session | None, default: None ) –

    Session to use for the chain.

  • settings (dict | None, default: None ) –

    Settings to use for the chain.

  • column_types (dict[str, str | DataType] | None, default: None ) –

    Dictionary of column names and their corresponding types. It is passed to CSV reader and for each column specified type auto inference is disabled.

  • parse_options (dict[str, str | bool | Callable] | None, default: None ) –

    Tells the parser how to process lines. See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

Example

Reading a csv file: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/file.csv")

Reading csv files from a directory as a combined dataset: 

import datachain as dc
chain = dc.read_csv("s3://mybucket/dir")

Source code in datachain/lib/dc/csv.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def read_csv(
    path: str | os.PathLike[str] | list[str] | list[os.PathLike[str]],
    delimiter: str | None = None,
    header: bool = True,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: int | None = None,
    session: Session | None = None,
    settings: dict | None = None,
    column_types: dict[str, "str | ArrowDataType"] | None = None,
    parse_options: dict[str, str | bool | Callable] | None = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from csv files.

    Parameters:
        path: Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        delimiter: Character for delimiting columns. Takes precedence if also
            specified in `parse_options`. Defaults to ",".
        header: Whether the files include a header row.
        output: Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column: Created column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        nrows: Optional row limit.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        column_types: Dictionary of column names and their corresponding types.
            It is passed to CSV reader and for each column specified type auto
            inference is disabled.
        parse_options: Tells the parser how to process lines.
            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

    Example:
        Reading a csv file:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/file.csv")
        ```

        Reading csv files from a directory as a combined dataset:
        ```py
        import datachain as dc
        chain = dc.read_csv("s3://mybucket/dir")
        ```
    """
    from pandas._libs.parsers import STR_NA_VALUES
    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
    from pyarrow.dataset import CsvFileFormat
    from pyarrow.lib import type_for_alias

    from .storage import read_storage

    parse_options = parse_options or {}
    if "delimiter" not in parse_options:
        parse_options["delimiter"] = ","
    if delimiter:
        parse_options["delimiter"] = delimiter

    if column_types:
        column_types = {
            name: type_for_alias(typ) if isinstance(typ, str) else typ
            for name, typ in column_types.items()
        }
    else:
        column_types = {}

    chain = read_storage(path, session=session, settings=settings, **kwargs)

    column_names = None
    if not header:
        if not output:
            msg = "error parsing csv - provide output if no header"
            raise DatasetPrepareError(chain.name, msg)
        if isinstance(output, Sequence):
            column_names = output  # type: ignore[assignment]
        elif isinstance(output, dict):
            column_names = list(output.keys())
        elif (fr := ModelStore.to_pydantic(output)) is not None:
            column_names = list(fr.model_fields.keys())
        else:
            msg = f"error parsing csv - incompatible output type {type(output)}"
            raise DatasetPrepareError(chain.name, msg)

    parse_options = ParseOptions(**parse_options)
    read_options = ReadOptions(column_names=column_names)
    convert_options = ConvertOptions(
        strings_can_be_null=True,
        null_values=STR_NA_VALUES,
        column_types=column_types,
    )
    format = CsvFileFormat(
        parse_options=parse_options,
        read_options=read_options,
        convert_options=convert_options,
    )
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        nrows=nrows,
        format=format,
        parse_options=parse_options,
    )

read_dataset 

read_dataset(
    name: str,
    namespace: str | None = None,
    project: str | None = None,
    version: str | int | None = None,
    session: Session | None = None,
    settings: dict | None = None,
    delta: bool | None = False,
    delta_on: str | Sequence[str] | None = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: str | Sequence[str] | None = None,
    delta_compare: str | Sequence[str] | None = None,
    delta_retry: bool | str | None = None,
    delta_unsafe: bool = False,
    update: bool = False,
) -> DataChain

Get data from a saved Dataset. It returns the chain itself. If dataset or version is not found locally, it will try to pull it from Studio.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace (str | None, default: None ) –

    optional name of namespace in which dataset to read is created

  • project (str | None, default: None ) –

    optional name of project in which dataset to read is created

  • version (str | int | None, default: None ) –

    dataset version. Supports: - Exact version strings: "1.2.3" - Legacy integer versions: 1, 2, 3 (finds latest major version) - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.

  • session (Session | None, default: None ) –

    Session to use for the chain.

  • settings (dict | None, default: None ) –

    Settings to use for the chain.

  • delta (bool | None, default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (str | Sequence[str] | None, default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (str | Sequence[str] | None, default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (str | Sequence[str] | None, default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (bool | str | None, default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

  • update (bool, default: False ) –

    If True always checks for newer versions available on Studio, even if some version of the dataset exists locally already. If False (default), it will only fetch the dataset from Studio if it is not found locally.

  • delta_unsafe (bool, default: False ) –

    Allow restricted ops in delta: merge, agg, union, group_by, distinct.

Example 
import datachain as dc
chain = dc.read_dataset("my_cats")

import datachain as dc
chain = dc.read_dataset("dev.animals.my_cats")

chain = dc.read_dataset("my_cats", version="1.0.0")

# Using version specifiers (PEP 440)
chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")

# Legacy integer version support (finds latest in major version)
chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version

# Always check for newer versions matching a version specifier from Studio
chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)

session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
settings = {
    "cache": True,
    "parallel": 4,
    "workers": 4,
    "min_task_size": 1000,
    "prefetch": 10,
}
chain = dc.read_dataset(
    name="my_cats",
    version="1.0.0",
    session=session,
    settings=settings,
)
Source code in datachain/lib/dc/datasets.py 
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def read_dataset(
    name: str,
    namespace: str | None = None,
    project: str | None = None,
    version: str | int | None = None,
    session: Session | None = None,
    settings: dict | None = None,
    delta: bool | None = False,
    delta_on: str | Sequence[str] | None = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: str | Sequence[str] | None = None,
    delta_compare: str | Sequence[str] | None = None,
    delta_retry: bool | str | None = None,
    delta_unsafe: bool = False,
    update: bool = False,
) -> "DataChain":
    """Get data from a saved Dataset. It returns the chain itself.
    If dataset or version is not found locally, it will try to pull it from Studio.

    Parameters:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace: optional name of namespace in which dataset to read is created
        project: optional name of project in which dataset to read is created
        version: dataset version. Supports:
            - Exact version strings: "1.2.3"
            - Legacy integer versions: 1, 2, 3 (finds latest major version)
            - Version specifiers (PEP 440): ">=1.0.0,<2.0.0", "~=1.4.2", "==1.2.*", etc.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)
        update: If True always checks for newer versions available on Studio, even if
            some version of the dataset exists locally already. If False (default), it
            will only fetch the dataset from Studio if it is not found locally.
        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
            distinct.


    Example:
        ```py
        import datachain as dc
        chain = dc.read_dataset("my_cats")
        ```

        ```py
        import datachain as dc
        chain = dc.read_dataset("dev.animals.my_cats")
        ```

        ```py
        chain = dc.read_dataset("my_cats", version="1.0.0")
        ```

        ```py
        # Using version specifiers (PEP 440)
        chain = dc.read_dataset("my_cats", version=">=1.0.0,<2.0.0")
        ```

        ```py
        # Legacy integer version support (finds latest in major version)
        chain = dc.read_dataset("my_cats", version=1)  # Latest 1.x.x version
        ```

        ```py
        # Always check for newer versions matching a version specifier from Studio
        chain = dc.read_dataset("my_cats", version=">=1.0.0", update=True)
        ```

        ```py
        session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
        settings = {
            "cache": True,
            "parallel": 4,
            "workers": 4,
            "min_task_size": 1000,
            "prefetch": 10,
        }
        chain = dc.read_dataset(
            name="my_cats",
            version="1.0.0",
            session=session,
            settings=settings,
        )
        ```
    """
    from datachain.telemetry import telemetry

    from .datachain import DataChain

    telemetry.send_event_once("class", "datachain_init", name=name, version=version)

    session = Session.get(session)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if version is not None:
        dataset = session.catalog.get_dataset_with_remote_fallback(
            name, namespace_name, project_name, update=update
        )

        # Convert legacy integer versions to version specifiers
        # For backward compatibility we still allow users to put version as integer
        # in which case we convert it to a version specifier that finds the latest
        # version where major part is equal to that input version.
        # For example if user sets version=2, we convert it to ">=2.0.0,<3.0.0"
        # which will find something like 2.4.3 (assuming 2.4.3 is the biggest among
        # all 2.* dataset versions)
        if isinstance(version, int):
            version_spec = f">={version}.0.0,<{version + 1}.0.0"
        else:
            version_spec = str(version)

        from packaging.specifiers import InvalidSpecifier, SpecifierSet

        try:
            # Try to parse as version specifier
            SpecifierSet(version_spec)
            # If it's a valid specifier set, find the latest compatible version
            latest_compatible = dataset.latest_compatible_version(version_spec)
            if not latest_compatible:
                raise DatasetVersionNotFoundError(
                    f"No dataset {name} version matching specifier {version_spec}"
                )
            version = latest_compatible
        except InvalidSpecifier:
            # If not a valid specifier, treat as exact version string
            # This handles cases like "1.2.3" which are exact versions, not specifiers
            pass

    if settings:
        _settings = Settings(**settings)
    else:
        _settings = Settings()

    query = DatasetQuery(
        name=name,
        project_name=project_name,
        namespace_name=namespace_name,
        version=version,  #  type: ignore[arg-type]
        session=session,
        update=update,
    )

    signals_schema = SignalSchema({"sys": Sys})
    if query.feature_schema:
        signals_schema |= SignalSchema.deserialize(query.feature_schema)
    else:
        signals_schema |= SignalSchema.from_column_types(query.column_types or {})
    chain = DataChain(query, _settings, signals_schema)

    if delta:
        chain = chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
            delta_unsafe=delta_unsafe,
        )

    return chain

datasets 

datasets(
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    column: str | None = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: list[str] | None = None,
) -> DataChain

Generate chain with list of registered datasets.

Parameters:

  • session (Session | None, default: None ) –

    Optional session instance. If not provided, uses default session.

  • settings (dict | None, default: None ) –

    Optional dictionary of settings to configure the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • column (str | None, default: None ) –

    Name of the output column in the chain. Defaults to None which means no top level column will be created.

  • include_listing (bool, default: False ) –

    If True, includes listing datasets. Defaults to False.

  • studio (bool, default: False ) –

    If True, returns datasets from Studio only, otherwise returns all local datasets. Defaults to False.

  • attrs (list[str] | None, default: None ) –

    Optional list of attributes to filter datasets on. It can be just attribute without value e.g "NLP", or attribute with value e.g "location=US". Attribute with value can also accept "" to target all that have specific name e.g "location="

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance containing dataset information.

Example 
import datachain as dc

chain = dc.datasets(column="dataset")
for ds in chain.to_iter("dataset"):
    print(f"{ds.name}@v{ds.version}")
Source code in datachain/lib/dc/datasets.py 
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def datasets(
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    column: str | None = None,
    include_listing: bool = False,
    studio: bool = False,
    attrs: list[str] | None = None,
) -> "DataChain":
    """Generate chain with list of registered datasets.

    Args:
        session: Optional session instance. If not provided, uses default session.
        settings: Optional dictionary of settings to configure the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        column: Name of the output column in the chain. Defaults to None which
            means no top level column will be created.
        include_listing: If True, includes listing datasets. Defaults to False.
        studio: If True, returns datasets from Studio only,
            otherwise returns all local datasets. Defaults to False.
        attrs: Optional list of attributes to filter datasets on. It can be just
            attribute without value e.g "NLP", or attribute with value
            e.g "location=US". Attribute with value can also accept "*" to target
            all that have specific name e.g "location=*"

    Returns:
        DataChain: A new DataChain instance containing dataset information.

    Example:
        ```py
        import datachain as dc

        chain = dc.datasets(column="dataset")
        for ds in chain.to_iter("dataset"):
            print(f"{ds.name}@v{ds.version}")
        ```
    """

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    datasets_values = [
        DatasetInfo.from_models(d, v, j)
        for d, v, j in catalog.list_datasets_versions(
            include_listing=include_listing, studio=studio
        )
    ]
    datasets_values = [d for d in datasets_values if not d.is_temp]

    if attrs:
        for attr in attrs:
            datasets_values = [d for d in datasets_values if d.has_attr(attr)]

    if not column:
        # flattening dataset fields
        schema = {
            k: get_origin(v) if get_origin(v) is dict else v
            for k, v in get_type_hints(DatasetInfo).items()
            if k in DatasetInfo.model_fields
        }
        data = {k: [] for k in DatasetInfo.model_fields}  # type: ignore[var-annotated]
        for d in [d.model_dump() for d in datasets_values]:
            for field, value in d.items():
                data[field].append(value)

        return read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            output=schema,
            **data,  # type: ignore[arg-type]
        )

    return read_values(
        session=session,
        settings=settings,
        in_memory=in_memory,
        output={column: DatasetInfo},
        **{column: datasets_values},  # type: ignore[arg-type]
    )

delete_dataset 

delete_dataset(
    name: str,
    namespace: str | None = None,
    project: str | None = None,
    version: str | None = None,
    force: bool | None = False,
    studio: bool | None = False,
    session: Session | None = None,
    in_memory: bool = False,
) -> None

Removes specific dataset version or all dataset versions, depending on a force flag.

Parameters:

  • name (str) –

    The dataset name, which can be a fully qualified name including the namespace and project. Alternatively, it can be a regular name, in which case the explicitly defined namespace and project will be used if they are set; otherwise, default values will be applied.

  • namespace (str | None, default: None ) –

    optional name of namespace in which dataset to delete is created

  • project (str | None, default: None ) –

    optional name of project in which dataset to delete is created

  • version (str | None, default: None ) –

    Optional dataset version

  • force (bool | None, default: False ) –

    If true, all datasets versions will be removed. Defaults to False.

  • studio (bool | None, default: False ) –

    If True, removes dataset from Studio only, otherwise removes local dataset. Defaults to False.

  • session (Session | None, default: None ) –

    Optional session instance. If not provided, uses default session.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns: None

Example 
import datachain as dc
dc.delete_dataset("cats")

import datachain as dc
dc.delete_dataset("cats", version="1.0.0")
Source code in datachain/lib/dc/datasets.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
def delete_dataset(
    name: str,
    namespace: str | None = None,
    project: str | None = None,
    version: str | None = None,
    force: bool | None = False,
    studio: bool | None = False,
    session: Session | None = None,
    in_memory: bool = False,
) -> None:
    """Removes specific dataset version or all dataset versions, depending on
    a force flag.

    Args:
        name: The dataset name, which can be a fully qualified name including the
            namespace and project. Alternatively, it can be a regular name, in which
            case the explicitly defined namespace and project will be used if they are
            set; otherwise, default values will be applied.
        namespace: optional name of namespace in which dataset to delete is created
        project: optional name of project in which dataset to delete is created
        version: Optional dataset version
        force: If true, all datasets versions will be removed. Defaults to False.
        studio: If True, removes dataset from Studio only, otherwise removes local
            dataset. Defaults to False.
        session: Optional session instance. If not provided, uses default session.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns: None

    Example:
        ```py
        import datachain as dc
        dc.delete_dataset("cats")
        ```

        ```py
        import datachain as dc
        dc.delete_dataset("cats", version="1.0.0")
        ```
    """
    from datachain.studio import remove_studio_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        project_name=project,
        namespace_name=namespace,
    )

    if not is_studio() and studio:
        return remove_studio_dataset(
            None, name, namespace_name, project_name, version=version, force=force
        )

    try:
        ds_project = get_project(project_name, namespace_name, session=session)
    except ProjectNotFoundError:
        raise DatasetNotFoundError(
            f"Dataset {name} not found in namespace {namespace_name} and project",
            f" {project_name}",
        ) from None

    if not force:
        version = (
            version
            or catalog.get_dataset(
                name,
                namespace_name=ds_project.namespace.name,
                project_name=ds_project.name,
            ).latest_version
        )
    else:
        version = None
    catalog.remove_dataset(name, ds_project, version=version, force=force)

move_dataset 

move_dataset(
    src: str,
    dest: str,
    session: Session | None = None,
    in_memory: bool = False,
) -> None

Moves an entire dataset between namespaces and projects.

Parameters:

  • src (str) –

    The source dataset name. This can be a fully qualified name that includes the namespace and project, or a regular name. If a regular name is used, default values will be applied. The source dataset will no longer exist after the move.

  • dest (str) –

    The destination dataset name. This can also be a fully qualified name with a namespace and project, or just a regular name (default values will be used in that case). The original dataset will be moved here.

  • session (Session | None, default: None ) –

    An optional session instance. If not provided, the default session will be used.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

Returns:

  • None –

    None

Examples:

import datachain as dc
dc.move_dataset("cats", "new_cats")

import datachain as dc
dc.move_dataset("dev.animals.cats", "prod.animals.cats")
Source code in datachain/lib/dc/datasets.py 
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def move_dataset(
    src: str,
    dest: str,
    session: Session | None = None,
    in_memory: bool = False,
) -> None:
    """Moves an entire dataset between namespaces and projects.

    Args:
        src: The source dataset name. This can be a fully qualified name that includes
            the namespace and project, or a regular name. If a regular name is used,
            default values will be applied. The source dataset will no longer exist
            after the move.
        dest: The destination dataset name. This can also be a fully qualified
            name with a namespace and project, or just a regular name (default values
            will be used in that case). The original dataset will be moved here.
        session: An optional session instance. If not provided, the default session
            will be used.
        in_memory: If True, creates an in-memory session. Defaults to False.

    Returns:
        None

    Examples:
        ```python
        import datachain as dc
        dc.move_dataset("cats", "new_cats")
        ```

        ```python
        import datachain as dc
        dc.move_dataset("dev.animals.cats", "prod.animals.cats")
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    namespace, project, name = catalog.get_full_dataset_name(src)
    dest_namespace, dest_project, dest_name = catalog.get_full_dataset_name(dest)

    dataset = catalog.get_dataset(name, namespace_name=namespace, project_name=project)

    catalog.update_dataset(
        dataset,
        name=dest_name,
        project_id=catalog.metastore.get_project(
            dest_project,
            dest_namespace,
            create=is_studio(),
        ).id,
    )

delete_namespace 

delete_namespace(
    name: str, session: Session | None = None
) -> None

Removes a namespace by name.

Raises:

  • NamespaceNotFoundError –

    If the namespace does not exist.

  • NamespaceDeleteNotAllowedError –

    If the namespace is non-empty, is the default namespace, or is a system namespace, as these cannot be removed.

Parameters:

  • name (str) –

    The name of the namespace.

  • session (Session | None, default: None ) –

    Session to use for getting project.

Example 
import datachain as dc
dc.delete_namespace("dev")
Source code in datachain/lib/namespaces.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def delete_namespace(name: str, session: Session | None = None) -> None:
    """
    Removes a namespace by name.

    Raises:
        NamespaceNotFoundError: If the namespace does not exist.
        NamespaceDeleteNotAllowedError: If the namespace is non-empty,
            is the default namespace, or is a system namespace,
            as these cannot be removed.

    Parameters:
        name: The name of the namespace.
        session: Session to use for getting project.

    Example:
        ```py
        import datachain as dc
        dc.delete_namespace("dev")
        ```
    """
    session = Session.get(session)
    metastore = session.catalog.metastore

    namespace_name, project_name = parse_name(name)

    if project_name:
        return delete_project(project_name, namespace_name, session)

    namespace = metastore.get_namespace(name)

    if name == metastore.system_namespace_name:
        raise NamespaceDeleteNotAllowedError(
            f"Namespace {metastore.system_namespace_name} cannot be removed"
        )

    if name == metastore.default_namespace_name:
        raise NamespaceDeleteNotAllowedError(
            f"Namespace {metastore.default_namespace_name} cannot be removed"
        )

    num_projects = metastore.count_projects(namespace.id)
    if num_projects > 0:
        raise NamespaceDeleteNotAllowedError(
            f"Namespace cannot be removed. It contains {num_projects} project(s). "
            "Please remove the project(s) first."
        )

    metastore.remove_namespace(namespace.id)

read_hf 

read_hf(
    dataset: HFDatasetType,
    *args: Any,
    session: Session | None = None,
    settings: dict | None = None,
    column: str = "",
    model_name: str = "",
    limit: int = 0,
    **kwargs: Any
) -> DataChain

Generate chain from Hugging Face Hub dataset.

Parameters:

  • dataset (HFDatasetType) –

    Path or name of the dataset to read from Hugging Face Hub, or an instance of datasets.Dataset-like object.

  • args (Any, default: () ) –

    Additional positional arguments to pass to datasets.load_dataset.

  • session (Session | None, default: None ) –

    Session to use for the chain.

  • settings (dict | None, default: None ) –

    Settings to use for the chain.

  • column (str, default: '' ) –

    Generated object column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • limit (int, default: 0 ) –

    The maximum number of items to read from the HF dataset. Applies take(limit) to datasets.load_dataset. Defaults to 0 (no limit).

  • kwargs (Any, default: {} ) –

    Parameters to pass to datasets.load_dataset.

Example

Load from Hugging Face Hub: 

import datachain as dc
chain = dc.read_hf("beans", split="train")

Generate chain from loaded dataset: 

from datasets import load_dataset
ds = load_dataset("beans", split="train")
import datachain as dc
chain = dc.read_hf(ds)

Streaming with limit, for large datasets: 

import datachain as dc
ds = dc.read_hf("beans", split="train", streaming=True, limit=10)

or use HF split syntax (not supported if streaming is enabled): 

import datachain as dc
ds = dc.read_hf("beans", split="train[%10]")

Source code in datachain/lib/dc/hf.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def read_hf(
    dataset: "HFDatasetType",
    *args: Any,
    session: Session | None = None,
    settings: dict | None = None,
    column: str = "",
    model_name: str = "",
    limit: int = 0,
    **kwargs: Any,
) -> "DataChain":
    """Generate chain from Hugging Face Hub dataset.

    Parameters:
        dataset: Path or name of the dataset to read from Hugging Face Hub,
            or an instance of `datasets.Dataset`-like object.
        args: Additional positional arguments to pass to `datasets.load_dataset`.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        column: Generated object column name.
        model_name: Generated model name.
        limit: The maximum number of items to read from the HF dataset.
            Applies `take(limit)` to `datasets.load_dataset`.
            Defaults to 0 (no limit).
        kwargs: Parameters to pass to `datasets.load_dataset`.

    Example:
        Load from Hugging Face Hub:
        ```py
        import datachain as dc
        chain = dc.read_hf("beans", split="train")
        ```

        Generate chain from loaded dataset:
        ```py
        from datasets import load_dataset
        ds = load_dataset("beans", split="train")
        import datachain as dc
        chain = dc.read_hf(ds)
        ```

        Streaming with limit, for large datasets:
        ```py
        import datachain as dc
        ds = dc.read_hf("beans", split="train", streaming=True, limit=10)
        ```

        or use HF split syntax (not supported if streaming is enabled):
        ```py
        import datachain as dc
        ds = dc.read_hf("beans", split="train[%10]")
        ```
    """
    from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits

    from .values import read_values

    output: dict[str, DataType] = {}
    ds_dict = stream_splits(dataset, *args, **kwargs)
    if len(ds_dict) > 1:
        output = {"split": str}

    model_name = model_name or column or ""
    hf_features = next(iter(ds_dict.values())).features
    hf_output, normalized_names = get_output_schema(hf_features, list(output.keys()))
    output = output | hf_output
    model = dict_to_data_model(model_name, output, list(normalized_names.values()))
    if column:
        output = {column: model}

    chain = read_values(split=list(ds_dict.keys()), session=session, settings=settings)
    return chain.gen(HFGenerator(dataset, model, limit, *args, **kwargs), output=output)

read_json 

read_json(
    path: str | PathLike[str],
    type: FileType = "text",
    spec: DataType | None = None,
    schema_from: str | None = "auto",
    jmespath: str | None = None,
    column: str | None = "",
    model_name: str | None = None,
    format: str | None = "json",
    nrows: int | None = None,
    **kwargs
) -> DataChain

Get data from JSON. It returns the chain itself.

Parameters:

  • path (str | PathLike[str]) –

    storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type (FileType, default: 'text' ) –

    read file as "binary", "text", or "image" data. Default is "text".

  • spec (DataType | None, default: None ) –

    optional Data Model

  • schema_from (str | None, default: 'auto' ) –

    path to sample to infer spec (if schema not provided)

  • column (str | None, default: '' ) –

    generated column name

  • model_name (str | None, default: None ) –

    optional generated model name

  • format (str | None, default: 'json' ) –

    "json", "jsonl"

  • jmespath (str | None, default: None ) –

    optional JMESPATH expression to reduce JSON

  • nrows (int | None, default: None ) –

    optional row limit for jsonl and JSON arrays

Example

infer JSON schema from data, reduce using JMESPATH 

import datachain as dc
chain = dc.read_json("gs://json", jmespath="key1.key2")

infer JSON schema from a particular path 

import datachain as dc
chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")

Source code in datachain/lib/dc/json.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def read_json(
    path: str | os.PathLike[str],
    type: FileType = "text",
    spec: DataType | None = None,
    schema_from: str | None = "auto",
    jmespath: str | None = None,
    column: str | None = "",
    model_name: str | None = None,
    format: str | None = "json",
    nrows: int | None = None,
    **kwargs,
) -> "DataChain":
    """Get data from JSON. It returns the chain itself.

    Parameters:
        path: storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type: read file as "binary", "text", or "image" data. Default is "text".
        spec: optional Data Model
        schema_from: path to sample to infer spec (if schema not provided)
        column: generated column name
        model_name: optional generated model name
        format: "json", "jsonl"
        jmespath: optional JMESPATH expression to reduce JSON
        nrows: optional row limit for jsonl and JSON arrays

    Example:
        infer JSON schema from data, reduce using JMESPATH
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json", jmespath="key1.key2")
        ```

        infer JSON schema from a particular path
        ```py
        import datachain as dc
        chain = dc.read_json("gs://json_ds", schema_from="gs://json/my.json")
        ```
    """
    from .storage import read_storage

    if schema_from == "auto":
        schema_from = os.fspath(path)

    def jmespath_to_name(s: str):
        name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
        return s[:name_end]

    if (not column) and jmespath:
        column = jmespath_to_name(jmespath)
    if not column:
        column = format
    chain = read_storage(uri=path, type=type, **kwargs)
    signal_dict = {
        column: meta_formats.read_meta(
            schema_from=schema_from,
            format=format,
            spec=spec,
            model_name=model_name,
            jmespath=jmespath,
            nrows=nrows,
        ),
        "params": {"file": File},
    }
    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}

    cloudpickle.register_pickle_by_value(meta_formats)

    return chain.settings(**settings).gen(**signal_dict)  # type: ignore[misc, arg-type]

listings 

listings(
    session: Session | None = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs
) -> DataChain

Generate chain with list of cached listings. Listing is a special kind of dataset which has directory listing data of some underlying storage (e.g S3 bucket).

Example 
import datachain as dc
dc.listings().show()
Source code in datachain/lib/dc/listings.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def listings(
    session: Session | None = None,
    in_memory: bool = False,
    column: str = "listing",
    **kwargs,
) -> "DataChain":
    """Generate chain with list of cached listings.
    Listing is a special kind of dataset which has directory listing data of
    some underlying storage (e.g S3 bucket).

    Example:
        ```py
        import datachain as dc
        dc.listings().show()
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = kwargs.get("catalog") or session.catalog

    return read_values(
        session=session,
        in_memory=in_memory,
        output={column: ListingInfo},
        **{column: catalog.listings()},  # type: ignore[arg-type]
    )

read_pandas 

read_pandas(
    df: DataFrame,
    name: str = "",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    column: str = "",
) -> DataChain

Generate chain from pandas data-frame.

Example 
import pandas as pd
import datachain as dc

df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
dc.read_pandas(df)
Source code in datachain/lib/dc/pandas.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def read_pandas(  # type: ignore[override]
    df: "pd.DataFrame",
    name: str = "",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    column: str = "",
) -> "DataChain":
    """Generate chain from pandas data-frame.

    Example:
        ```py
        import pandas as pd
        import datachain as dc

        df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
        dc.read_pandas(df)
        ```
    """
    from .utils import DatasetPrepareError

    def get_col_name(col):
        if isinstance(col, tuple):
            # Join tuple elements with underscore for MultiIndex columns
            return "_".join(map(str, col)).lower()
        # Handle regular string column names
        return str(col).lower()

    fr_map = {get_col_name(col): df[col].tolist() for col in df.columns}

    for c in fr_map:
        if not c.isidentifier():
            raise DatasetPrepareError(
                name,
                f"import from pandas error - '{c}' cannot be a column name",
            )

    return read_values(
        name,
        session,
        settings=settings,
        column=column,
        in_memory=in_memory,
        **fr_map,
    )

read_parquet 

read_parquet(
    path: (
        str
        | PathLike[str]
        | list[str]
        | list[PathLike[str]]
    ),
    partitioning: Any = "hive",
    output: dict[str, DataType] | None = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Session | None = None,
    settings: dict | None = None,
    **kwargs
) -> DataChain

Generate chain from parquet files.

Parameters:

  • path (str | PathLike[str] | list[str] | list[PathLike[str]]) –

    Storage path(s) or URI(s). Can be a local path or start with a storage prefix like s3://, gs://, az://, hf:// or "file:///". Supports glob patterns: - * : wildcard - ** : recursive wildcard - ? : single character - {a,b} : brace expansion list - {1..9} : brace numeric or alphabetic range

  • partitioning (Any, default: 'hive' ) –

    Any pyarrow partitioning schema.

  • output (dict[str, DataType] | None, default: None ) –

    Dictionary defining column names and their corresponding types.

  • column (str, default: '' ) –

    Created column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • session (Session | None, default: None ) –

    Session to use for the chain.

  • settings (dict | None, default: None ) –

    Settings to use for the chain.

Example

Reading a single file: 

import datachain as dc
dc.read_parquet("s3://mybucket/file.parquet")

All files from a directory: 

dc.read_parquet("s3://mybucket/dir/")

Only parquet files from a directory, and all it's subdirectories: 

dc.read_parquet("s3://mybucket/dir/**/*.parquet")

Using filename patterns - numeric, list, starting with zeros: 

dc.read_parquet("s3://mybucket/202{1..4}/{yellow,green}-{01..12}.parquet")

Source code in datachain/lib/dc/parquet.py 
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
def read_parquet(
    path: str | os.PathLike[str] | list[str] | list[os.PathLike[str]],
    partitioning: Any = "hive",
    output: dict[str, DataType] | None = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    session: Session | None = None,
    settings: dict | None = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from parquet files.

    Parameters:
        path: Storage path(s) or URI(s). Can be a local path or start with a
            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
            Supports glob patterns:
              - `*` : wildcard
              - `**` : recursive wildcard
              - `?` : single character
              - `{a,b}` : brace expansion list
              - `{1..9}` : brace numeric or alphabetic range
        partitioning: Any pyarrow partitioning schema.
        output: Dictionary defining column names and their corresponding types.
        column: Created column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        session: Session to use for the chain.
        settings: Settings to use for the chain.

    Example:
        Reading a single file:
        ```py
        import datachain as dc
        dc.read_parquet("s3://mybucket/file.parquet")
        ```

        All files from a directory:
        ```py
        dc.read_parquet("s3://mybucket/dir/")
        ```

        Only parquet files from a directory, and all it's subdirectories:
        ```py
        dc.read_parquet("s3://mybucket/dir/**/*.parquet")
        ```

        Using filename patterns - numeric, list, starting with zeros:
        ```py
        dc.read_parquet("s3://mybucket/202{1..4}/{yellow,green}-{01..12}.parquet")
        ```
    """
    from .storage import read_storage

    chain = read_storage(path, session=session, settings=settings, **kwargs)
    return chain.parse_tabular(
        output=output,
        column=column,
        model_name=model_name,
        source=source,
        format="parquet",
        partitioning=partitioning,
    )

read_records 

read_records(
    to_insert: dict | Iterable[dict] | None,
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    schema: dict[str, DataType] | None = None,
) -> DataChain

Create a DataChain from the provided records. This method can be used for programmatically generating a chain in contrast of reading data from storages or other sources.

Parameters:

  • to_insert (dict | Iterable[dict] | None) –

    records (or a single record) to insert. Each record is a dictionary of signals and their values.

  • schema (dict[str, DataType] | None, default: None ) –

    describes chain signals and their corresponding types

Example 
import datachain as dc
single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
Notes

This call blocks until all records are inserted.

Source code in datachain/lib/dc/records.py 
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def read_records(
    to_insert: dict | Iterable[dict] | None,
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    schema: dict[str, DataType] | None = None,
) -> "DataChain":
    """Create a DataChain from the provided records. This method can be used for
    programmatically generating a chain in contrast of reading data from storages
    or other sources.

    Parameters:
        to_insert: records (or a single record) to insert. Each record is
                    a dictionary of signals and their values.
        schema: describes chain signals and their corresponding types

    Example:
        ```py
        import datachain as dc
        single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
        ```

    Notes:
        This call blocks until all records are inserted.
    """
    from datachain.query.dataset import adjust_outputs, get_col_types
    from datachain.sql.types import SQLType

    from .datasets import read_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    name = session.generate_temp_dataset_name()
    signal_schema = None
    columns: list[sqlalchemy.Column] = []

    if schema:
        signal_schema = SignalSchema(schema)
        columns = [
            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
            for c in signal_schema.db_signals(as_columns=True)
        ]
    else:
        columns = [
            sqlalchemy.Column(name, typ)
            for name, typ in File._datachain_column_types.items()
        ]

    dsr = catalog.create_dataset(
        name,
        catalog.metastore.default_project,
        columns=columns,
        feature_schema=(
            signal_schema.clone_without_sys_signals().serialize()
            if signal_schema
            else None
        ),
    )

    session.add_dataset_version(dsr, dsr.latest_version)

    if isinstance(to_insert, dict):
        to_insert = [to_insert]
    elif not to_insert:
        to_insert = []

    warehouse = catalog.warehouse
    dr = warehouse.dataset_rows(dsr)
    table = dr.get_table()

    # Optimization: Compute row types once, rather than for every row.
    col_types = get_col_types(
        warehouse,
        {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
    )
    records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
    warehouse.insert_rows(table, records, batch_size=READ_RECORDS_BATCH_SIZE)
    warehouse.insert_rows_done(table)
    return read_dataset(name=dsr.full_name, session=session, settings=settings)

read_storage 

read_storage(
    uri: (
        str
        | PathLike[str]
        | list[str]
        | list[PathLike[str]]
    ),
    *,
    type: FileType = "binary",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    recursive: bool | None = True,
    column: str = "file",
    update: bool = False,
    anon: bool | None = None,
    delta: bool | None = False,
    delta_on: str | Sequence[str] | None = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: str | Sequence[str] | None = None,
    delta_compare: str | Sequence[str] | None = None,
    delta_retry: bool | str | None = None,
    delta_unsafe: bool = False,
    client_config: dict | None = None
) -> DataChain

Get data from storage(s) as a list of file with all file attributes. It returns the chain itself as usual.

Parameters:

  • uri (str | PathLike[str] | list[str] | list[PathLike[str]]) –

    Storage path(s) or URI(s). Can be a local path or start with a storage prefix like s3://, gs://, az://, hf:// or "file:///". Supports glob patterns: - * : wildcard - ** : recursive wildcard - ? : single character - {a,b} : brace expansion list - {1..9} : brace numeric or alphabetic range

  • type (FileType, default: 'binary' ) –

    read file as "binary", "text", or "image" data. Default is "binary".

  • recursive (bool | None, default: True ) –

    search recursively for the given path.

  • column (str, default: 'file' ) –

    Column name that will contain File objects. Default is "file".

  • update (bool, default: False ) –

    force storage reindexing. Default is False.

  • anon (bool | None, default: None ) –

    If True, we will treat cloud bucket as public one.

  • client_config (dict | None, default: None ) –

    Optional client configuration for the storage client.

  • delta (bool | None, default: False ) –

    If True, only process new or changed files instead of reprocessing everything. This saves time by skipping files that were already processed in previous versions. The optimization is working when a new version of the dataset is created. Default is False.

  • delta_on (str | Sequence[str] | None, default: ('file.path', 'file.etag', 'file.version') ) –

    Field(s) that uniquely identify each record in the source data. Used to detect which records are new or changed. Default is ("file.path", "file.etag", "file.version").

  • delta_result_on (str | Sequence[str] | None, default: None ) –

    Field(s) in the result dataset that match delta_on fields. Only needed if you rename the identifying fields during processing. Default is None.

  • delta_compare (str | Sequence[str] | None, default: None ) –

    Field(s) used to detect if a record has changed. If not specified, all fields except delta_on fields are used. Default is None.

  • delta_retry (bool | str | None, default: None ) –

    Controls retry behavior for failed records: - String (field name): Reprocess records where this field is not empty (error mode) - True: Reprocess records missing from the result dataset (missing mode) - None: No retry processing (default)

  • delta_unsafe (bool, default: False ) –

    Allow restricted ops in delta: merge, agg, union, group_by, distinct. Caller must ensure datasets are consistent and not partially updated.

Returns:

  • DataChain ( DataChain ) –

    A DataChain object containing the file information.

Examples:

Simple call from s3: 

import datachain as dc
dc.read_storage("s3://my-bucket/my-dir")

Match all .json files recursively using glob pattern 

dc.read_storage("gs://bucket/meta/**/*.json")

Match image file extensions for directories with pattern 

dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")

By ranges in filenames: 

dc.read_storage("s3://bucket/202{1..4}/**/*.{jpg,jpeg,png}")

Multiple URIs: 

dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])

With AWS S3-compatible storage: 

dc.read_storage(
    "s3://my-bucket/my-dir",
    client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
)

Source code in datachain/lib/dc/storage.py 
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
def read_storage(
    uri: str | os.PathLike[str] | list[str] | list[os.PathLike[str]],
    *,
    type: FileType = "binary",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    recursive: bool | None = True,
    column: str = "file",
    update: bool = False,
    anon: bool | None = None,
    delta: bool | None = False,
    delta_on: str | Sequence[str] | None = (
        "file.path",
        "file.etag",
        "file.version",
    ),
    delta_result_on: str | Sequence[str] | None = None,
    delta_compare: str | Sequence[str] | None = None,
    delta_retry: bool | str | None = None,
    delta_unsafe: bool = False,
    client_config: dict | None = None,
) -> "DataChain":
    """Get data from storage(s) as a list of file with all file attributes.
    It returns the chain itself as usual.

    Parameters:
        uri: Storage path(s) or URI(s). Can be a local path or start with a
            storage prefix like `s3://`, `gs://`, `az://`, `hf://` or "file:///".
            Supports glob patterns:
              - `*` : wildcard
              - `**` : recursive wildcard
              - `?` : single character
              - `{a,b}` : brace expansion list
              - `{1..9}` : brace numeric or alphabetic range
        type: read file as "binary", "text", or "image" data. Default is "binary".
        recursive: search recursively for the given path.
        column: Column name that will contain File objects. Default is "file".
        update: force storage reindexing. Default is False.
        anon: If True, we will treat cloud bucket as public one.
        client_config: Optional client configuration for the storage client.
        delta: If True, only process new or changed files instead of reprocessing
            everything. This saves time by skipping files that were already processed in
            previous versions. The optimization is working when a new version of the
            dataset is created.
            Default is False.
        delta_on: Field(s) that uniquely identify each record in the source data.
            Used to detect which records are new or changed.
            Default is ("file.path", "file.etag", "file.version").
        delta_result_on: Field(s) in the result dataset that match `delta_on` fields.
            Only needed if you rename the identifying fields during processing.
            Default is None.
        delta_compare: Field(s) used to detect if a record has changed.
            If not specified, all fields except `delta_on` fields are used.
            Default is None.
        delta_retry: Controls retry behavior for failed records:
            - String (field name): Reprocess records where this field is not empty
              (error mode)
            - True: Reprocess records missing from the result dataset (missing mode)
            - None: No retry processing (default)
        delta_unsafe: Allow restricted ops in delta: merge, agg, union, group_by,
            distinct. Caller must ensure datasets are consistent and not partially
            updated.

    Returns:
        DataChain: A DataChain object containing the file information.

    Examples:
        Simple call from s3:
        ```python
        import datachain as dc
        dc.read_storage("s3://my-bucket/my-dir")
        ```

        Match all .json files recursively using glob pattern
        ```py
        dc.read_storage("gs://bucket/meta/**/*.json")
        ```

        Match image file extensions for directories with pattern
        ```py
        dc.read_storage("s3://bucket/202?/**/*.{jpg,jpeg,png}")
        ```

        By ranges in filenames:
        ```py
        dc.read_storage("s3://bucket/202{1..4}/**/*.{jpg,jpeg,png}")
        ```

        Multiple URIs:
        ```python
        dc.read_storage(["s3://my-bkt/dir1", "s3://bucket2/dir2/dir3"])
        ```

        With AWS S3-compatible storage:
        ```python
        dc.read_storage(
            "s3://my-bucket/my-dir",
            client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
        )
        ```
    """
    from .datachain import DataChain
    from .datasets import read_dataset
    from .records import read_records
    from .values import read_values

    file_type = get_file_type(type)

    if anon is not None:
        client_config = (client_config or {}) | {"anon": anon}
    session = Session.get(session, client_config=client_config, in_memory=in_memory)
    catalog = session.catalog
    cache = catalog.cache
    client_config = session.catalog.client_config
    listing_namespace_name = catalog.metastore.system_namespace_name
    listing_project_name = catalog.metastore.listing_project_name

    uris = uri if isinstance(uri, (list, tuple)) else [uri]

    if not uris:
        raise ValueError("No URIs provided")

    # Then expand all URIs that contain brace patterns
    expanded_uris = []
    for single_uri in uris:
        uri_str = str(single_uri)
        validate_cloud_bucket_name(uri_str)
        expanded_uris.extend(expand_brace_pattern(uri_str))

    # Now process each expanded URI
    chains = []
    listed_ds_name = set()
    file_values = []

    updated_uris = set()

    for single_uri in expanded_uris:
        # Check if URI contains glob patterns and split them
        base_uri, glob_pattern = split_uri_pattern(single_uri)

        # If a pattern is found, use the base_uri for listing
        # The pattern will be used for filtering later
        list_uri_to_use = base_uri if glob_pattern else single_uri

        # Avoid double updates for the same URI
        update_single_uri = False
        if update and (list_uri_to_use not in updated_uris):
            updated_uris.add(list_uri_to_use)
            update_single_uri = True

        list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
            list_uri_to_use, session, update=update_single_uri
        )

        # list_ds_name is None if object is a file, we don't want to use cache
        # or do listing in that case - just read that single object
        if not list_ds_name:
            file_values.append(
                get_file_info(list_uri, cache, client_config=client_config)
            )
            continue

        dc = read_dataset(
            list_ds_name,
            namespace=listing_namespace_name,
            project=listing_project_name,
            session=session,
            settings=settings,
        )
        dc._query.update = update
        dc.signals_schema = dc.signals_schema.mutate({f"{column}": file_type})

        if update or not list_ds_exists:

            def lst_fn(ds_name, lst_uri):
                # disable prefetch for listing, as it pre-downloads all files
                (
                    read_records(
                        DataChain.DEFAULT_FILE_RECORD,
                        session=session,
                        settings=settings,
                        in_memory=in_memory,
                    )
                    .settings(
                        prefetch=0,
                        namespace=listing_namespace_name,
                        project=listing_project_name,
                    )
                    .gen(
                        list_bucket(lst_uri, cache, client_config=client_config),
                        output={f"{column}": file_type},
                    )
                    # for internal listing datasets, we always bump major version
                    .save(ds_name, listing=True, update_version="major")
                )

            dc._query.set_listing_fn(
                lambda ds_name=list_ds_name, lst_uri=list_uri: lst_fn(ds_name, lst_uri)
            )

        # If a glob pattern was detected, use it for filtering
        # Otherwise, use the original list_path from get_listing
        if glob_pattern:
            # Determine if we should use recursive listing based on the pattern
            use_recursive = should_use_recursion(glob_pattern, recursive or False)

            # Apply glob filter - no need for brace expansion here as it's done above
            chain = apply_glob_filter(
                dc, glob_pattern, list_path, use_recursive, column
            )
            chains.append(chain)
        else:
            # No glob pattern detected, use normal ls behavior
            chains.append(ls(dc, list_path, recursive=recursive, column=column))

        listed_ds_name.add(list_ds_name)

    storage_chain = None if not chains else reduce(lambda x, y: x.union(y), chains)

    if file_values:
        file_chain = read_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            file=file_values,
        )
        file_chain.signals_schema = file_chain.signals_schema.mutate(
            {f"{column}": file_type}
        )
        storage_chain = storage_chain.union(file_chain) if storage_chain else file_chain

    assert storage_chain is not None

    if delta:
        storage_chain = storage_chain._as_delta(
            on=delta_on,
            right_on=delta_result_on,
            compare=delta_compare,
            delta_retry=delta_retry,
            delta_unsafe=delta_unsafe,
        )

    return storage_chain

read_values 

read_values(
    ds_name: str = "",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map
) -> DataChain

Generate chain from list of values.

Example 
import datachain as dc
dc.read_values(fib=[1, 2, 3, 5, 8])
Source code in datachain/lib/dc/values.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def read_values(
    ds_name: str = "",
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    output: OutputType = None,
    column: str = "",
    **fr_map,
) -> "DataChain":
    """Generate chain from list of values.

    Example:
        ```py
        import datachain as dc
        dc.read_values(fib=[1, 2, 3, 5, 8])
        ```
    """
    from .datachain import DataChain

    tuple_type, output, tuples = values_to_tuples(ds_name, output, **fr_map)

    def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
        yield from tuples

    _func_fr.__name__ = "read_values"

    chain = read_records(
        DataChain.DEFAULT_FILE_RECORD,
        session=session,
        settings=settings,
        in_memory=in_memory,
    )
    if column:
        output = {column: dict_to_data_model(column, output)}  # type: ignore[arg-type]
    return chain.gen(_func_fr, output=output)

read_database 

read_database(
    query: str | Executable,
    connection: ConnectionType,
    params: (
        Sequence[Mapping[str, Any]]
        | Mapping[str, Any]
        | None
    ) = None,
    *,
    output: dict[str, DataType] | None = None,
    session: Session | None = None,
    settings: dict | None = None,
    in_memory: bool = False,
    infer_schema_length: int | None = 100
) -> DataChain

Read the results of a SQL query into a DataChain, using a given database connection.

Parameters:

  • query (str | Executable) –

    The SQL query to execute. Can be a raw SQL string or a SQLAlchemy Executable object.

  • connection (ConnectionType) –

    SQLAlchemy connectable, str, or a sqlite3 connection Using SQLAlchemy makes it possible to use any DB supported by that library. If a DBAPI2 object, only sqlite3 is supported. The user is responsible for engine disposal and connection closure for the SQLAlchemy connectable; str connections are closed automatically.

  • params (Sequence[Mapping[str, Any]] | Mapping[str, Any] | None, default: None ) –

    Parameters to pass to execute method.

  • output (dict[str, DataType] | None, default: None ) –

    A dictionary mapping column names to types, used to override the schema inferred from the query results.

  • session (Session | None, default: None ) –

    Session to use for the chain.

  • settings (dict | None, default: None ) –

    Settings to use for the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • infer_schema_length (int | None, default: 100 ) –

    The maximum number of rows to scan for inferring schema. If set to None, the full data may be scanned. The rows used for schema inference are stored in memory, so large values can lead to high memory usage. Only applies if the output parameter is not set for the given column.

Examples:

Reading from a SQL query against a user-supplied connection: 

query = "SELECT key, value FROM tbl"
chain = dc.read_database(query, connection, output={"value": float})

Load data from a SQLAlchemy driver/engine: 

from sqlalchemy import create_engine
engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
chain = dc.read_database("select * from tbl", engine)

Load data from a parameterized SQLAlchemy query: 

query = "SELECT key, value FROM tbl WHERE value > :value"
dc.read_database(query, engine, params={"value": 50})

Notes
  • This function works with a variety of databases — including, but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake, provided the appropriate driver is installed.
  • This call is blocking, and will execute the query and return once the results are saved.
Source code in datachain/lib/dc/database.py 
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
def read_database(
    query: "str | sqlalchemy.sql.expression.Executable",
    connection: "ConnectionType",
    params: Sequence[Mapping[str, Any]] | Mapping[str, Any] | None = None,
    *,
    output: dict[str, "DataType"] | None = None,
    session: "Session | None" = None,
    settings: dict | None = None,
    in_memory: bool = False,
    infer_schema_length: int | None = 100,
) -> "DataChain":
    """
    Read the results of a SQL query into a DataChain, using a given database connection.

    Args:
        query:
            The SQL query to execute. Can be a raw SQL string or a SQLAlchemy
            `Executable` object.
        connection: SQLAlchemy connectable, str, or a sqlite3 connection
            Using SQLAlchemy makes it possible to use any DB supported by that
            library. If a DBAPI2 object, only sqlite3 is supported. The user is
            responsible for engine disposal and connection closure for the
            SQLAlchemy connectable; str connections are closed automatically.
        params: Parameters to pass to execute method.
        output: A dictionary mapping column names to types, used to override the
            schema inferred from the query results.
        session: Session to use for the chain.
        settings: Settings to use for the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        infer_schema_length:
            The maximum number of rows to scan for inferring schema.
            If set to `None`, the full data may be scanned.
            The rows used for schema inference are stored in memory,
            so large values can lead to high memory usage.
            Only applies if the `output` parameter is not set for the given column.

    Examples:
        Reading from a SQL query against a user-supplied connection:
        ```python
        query = "SELECT key, value FROM tbl"
        chain = dc.read_database(query, connection, output={"value": float})
        ```

        Load data from a SQLAlchemy driver/engine:
        ```python
        from sqlalchemy import create_engine
        engine = create_engine("postgresql+psycopg://myuser:mypassword@localhost:5432/mydb")
        chain = dc.read_database("select * from tbl", engine)
        ```

        Load data from a parameterized SQLAlchemy query:
        ```python
        query = "SELECT key, value FROM tbl WHERE value > :value"
        dc.read_database(query, engine, params={"value": 50})
        ```

    Notes:
        - This function works with a variety of databases — including,
        but not limited to, SQLite, DuckDB, PostgreSQL, and Snowflake,
        provided the appropriate driver is installed.
        - This call is blocking, and will execute the query and return once the
          results are saved.
    """
    from datachain.lib.dc.records import read_records

    output = output or {}
    if isinstance(query, str):
        query = sqlalchemy.text(query)
    kw = {"execution_options": {"stream_results": True}}  # use server-side cursors
    with _connect(connection) as conn, conn.execute(query, params, **kw) as result:
        cols = result.keys()
        to_infer = [k for k in cols if k not in output]  # preserve the order
        rows, inferred_schema = _infer_schema(result, to_infer, infer_schema_length)
        records = (row._asdict() for row in itertools.chain(rows, result))
        return read_records(
            records,
            session=session,
            settings=settings,
            in_memory=in_memory,
            schema=inferred_schema | output,
        )

ConnectionType module-attribute 

ConnectionType = (
    str
    | URL
    | Connectable
    | Engine
    | Connection
    | Session
    | Connection
)

DataChain 

DataChain(
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: dict | None = None,
    _sys: bool = False,
)

DataChain - a data structure for batch data processing and evaluation.

It represents a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data.

Data in DataChain is presented as Python classes with arbitrary set of fields, including nested classes. The data classes have to inherit from DataModel class. The supported set of field types include: majority of the type supported by the underlyind library Pydantic.

See Also

read_storage("s3://my-bucket/my-dir/") - reading unstructured data files from storages such as S3, gs or Azure ADLS.

DataChain.save("name") - saving to a dataset.

read_dataset("name") - reading from a dataset.

read_values(fib=[1, 2, 3, 5, 8]) - generating from values.

read_pandas(pd.DataFrame(...)) - generating from pandas.

read_json("file.json") - generating from json.

read_csv("file.csv") - generating from csv.

read_parquet("file.parquet") - generating from parquet.

Example 
import os

from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage
import datachain as dc

PROMPT = (
    "Was this bot dialog successful? "
    "Describe the 'result' as 'Yes' or 'No' in a short JSON"
)

model = "mistral-large-latest"
api_key = os.environ["MISTRAL_API_KEY"]

chain = (
    dc.read_storage("gs://datachain-demo/chatbot-KiT/")
    .limit(5)
    .settings(cache=True, parallel=5)
    .map(
        mistral_response=lambda file: MistralClient(api_key=api_key)
        .chat(
            model=model,
            response_format={"type": "json_object"},
            messages=[
                ChatMessage(role="user", content=f"{PROMPT}: {file.read()}")
            ],
        )
        .choices[0]
        .message.content,
    )
    .persist()
)

try:
    print(chain.select("mistral_response").results())
except Exception as e:
    print(f"do you have the right Mistral API key? {e}")
Source code in datachain/lib/dc/datachain.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
def __init__(
    self,
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: dict | None = None,
    _sys: bool = False,
) -> None:
    """Don't instantiate this directly, use one of the from_XXX constructors."""
    self._query = query
    self._settings = settings
    self.signals_schema = signal_schema
    self._setup: dict = setup or {}
    self._sys = _sys
    self._delta = False
    self._delta_unsafe = False
    self._delta_on: str | Sequence[str] | None = None
    self._delta_result_on: str | Sequence[str] | None = None
    self._delta_compare: str | Sequence[str] | None = None
    self._delta_retry: bool | str | None = None

dataset property 

dataset: DatasetRecord | None

Underlying dataset, if there is one.

delta property 

delta: bool

Returns True if this chain is ran in "delta" update mode

empty property 

empty: bool

Returns True if chain has zero number of rows

name property 

name: str | None

Name of the underlying dataset, if there is one.

namespace_name property 

namespace_name: str

Current namespace name in which the chain is running

project_name property 

project_name: str

Current project name in which the chain is running

schema property 

schema: dict[str, DataType]

Get schema of the chain.

session property 

session: Session

Session of the chain.

version property 

version: str | None

Version of the underlying dataset, if there is one.

__iter__ 

__iter__() -> Iterator[tuple[DataValue, ...]]

Make DataChain objects iterable.

Yields:

  • tuple[DataValue, ...] –

    Yields tuples of all column values for each row.

Example 
for row in chain:
    print(row)
Source code in datachain/lib/dc/datachain.py 
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
def __iter__(self) -> Iterator[tuple[DataValue, ...]]:
    """Make DataChain objects iterable.

    Yields:
        (tuple[DataValue, ...]): Yields tuples of all column values for each row.

    Example:
        ```py
        for row in chain:
            print(row)
        ```
    """
    return self.to_iter()

__or__ 

__or__(other: Self) -> Self

Return self.union(other).

Source code in datachain/lib/dc/datachain.py 
310
311
312
def __or__(self, other: "Self") -> "Self":
    """Return `self.union(other)`."""
    return self.union(other)

__repr__ 

__repr__() -> str

Return a string representation of the chain.

Source code in datachain/lib/dc/datachain.py 
203
204
205
206
207
208
209
210
211
212
213
def __repr__(self) -> str:
    """Return a string representation of the chain."""
    classname = self.__class__.__name__
    if not self._effective_signals_schema.values:
        return f"Empty {classname}"

    import io

    file = io.StringIO()
    self.print_schema(file=file)
    return file.getvalue()

agg 

agg(
    func: Callable | None = None,
    partition_by: PartitionByType | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map: Callable
) -> Self

Aggregate rows using partition_by statement and apply a function to the groups of aggregated rows. The function needs to return new objects for each group of the new rows. It returns a chain itself with new signals.

Input-output relationship: N:M

This method bears similarity to gen() and map(), employing a comparable set of parameters, yet differs in two crucial aspects:

  1. The partition_by parameter: This specifies the column name or a list of column names that determine the grouping criteria for aggregation.
  2. Group-based UDF function input: Instead of individual rows, the function receives a list of all rows within each group defined by partition_by.

If partition_by is not set or is an empty list, all rows will be placed into a single group.

Parameters:

  • func (Callable | None, default: None ) –

    Function applied to each group of rows.

  • partition_by (PartitionByType | None, default: None ) –

    Column name(s) to group by. If None, all rows go into one group.

  • params (str | Sequence[str] | None, default: None ) –

    List of column names used as input for the function. Default is taken from function signature.

  • output (OutputType, default: None ) –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature.

  • **signal_map (Callable, default: {} ) –

    kwargs can be used to define func together with its return signal name in format of agg(result_column=my_func).

Examples:

Basic aggregation with lambda function: 

chain = chain.agg(
    total=lambda category, amount: [sum(amount)],
    output=float,
    partition_by="category",
)
chain.save("new_dataset")

An alternative syntax, when you need to specify a more complex function: 

# It automatically resolves which columns to pass to the function
# by looking at the function signature.
def agg_sum(
    file: list[File], amount: list[float]
) -> Iterator[tuple[File, float]]:
    yield file[0], sum(amount)

chain = chain.agg(
    agg_sum,
    output={"file": File, "total": float},
    # Alternative syntax is to use `C` (short for Column) to specify
    # a column name or a nested column, e.g. C("file.path").
    partition_by=C("category"),
)
chain.save("new_dataset")

Using complex signals for partitioning (File or any Pydantic BaseModel): 

def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
    yield files[0], sum(f.size for f in files)

chain = chain.agg(
    my_agg,
    params=("file",),
    output={"file": File, "total": int},
    partition_by="file",  # Column referring to all sub-columns of File
)
chain.save("new_dataset")

Aggregating all rows into a single group (when partition_by is not set): 

chain = chain.agg(
    total_size=lambda file, size: [sum(size)],
    output=int,
    # No partition_by specified - all rows go into one group
)
chain.save("new_dataset")

Multiple partition columns: 

chain = chain.agg(
    total=lambda category, subcategory, amount: [sum(amount)],
    output=float,
    partition_by=["category", "subcategory"],
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
@delta_disabled
def agg(
    self,
    /,
    func: Callable | None = None,
    partition_by: PartitionByType | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map: Callable,
) -> "Self":
    """Aggregate rows using `partition_by` statement and apply a function to the
    groups of aggregated rows. The function needs to return new objects for each
    group of the new rows. It returns a chain itself with new signals.

    Input-output relationship: N:M

    This method bears similarity to `gen()` and `map()`, employing a comparable set
    of parameters, yet differs in two crucial aspects:

    1. The `partition_by` parameter: This specifies the column name or a list of
       column names that determine the grouping criteria for aggregation.
    2. Group-based UDF function input: Instead of individual rows, the function
       receives a list of all rows within each group defined by `partition_by`.

    If `partition_by` is not set or is an empty list, all rows will be placed
    into a single group.

    Parameters:
        func: Function applied to each group of rows.
        partition_by: Column name(s) to group by. If None, all rows go
            into one group.
        params: List of column names used as input for the function. Default is
            taken from function signature.
        output: Dictionary defining new signals and their corresponding types.
            Default type is taken from function signature.
        **signal_map: kwargs can be used to define `func` together with its return
            signal name in format of `agg(result_column=my_func)`.

    Examples:
        Basic aggregation with lambda function:
        ```py
        chain = chain.agg(
            total=lambda category, amount: [sum(amount)],
            output=float,
            partition_by="category",
        )
        chain.save("new_dataset")
        ```

        An alternative syntax, when you need to specify a more complex function:
        ```py
        # It automatically resolves which columns to pass to the function
        # by looking at the function signature.
        def agg_sum(
            file: list[File], amount: list[float]
        ) -> Iterator[tuple[File, float]]:
            yield file[0], sum(amount)

        chain = chain.agg(
            agg_sum,
            output={"file": File, "total": float},
            # Alternative syntax is to use `C` (short for Column) to specify
            # a column name or a nested column, e.g. C("file.path").
            partition_by=C("category"),
        )
        chain.save("new_dataset")
        ```

        Using complex signals for partitioning (`File` or any Pydantic `BaseModel`):
        ```py
        def my_agg(files: list[File]) -> Iterator[tuple[File, int]]:
            yield files[0], sum(f.size for f in files)

        chain = chain.agg(
            my_agg,
            params=("file",),
            output={"file": File, "total": int},
            partition_by="file",  # Column referring to all sub-columns of File
        )
        chain.save("new_dataset")
        ```

        Aggregating all rows into a single group (when `partition_by` is not set):
        ```py
        chain = chain.agg(
            total_size=lambda file, size: [sum(size)],
            output=int,
            # No partition_by specified - all rows go into one group
        )
        chain.save("new_dataset")
        ```

        Multiple partition columns:
        ```py
        chain = chain.agg(
            total=lambda category, subcategory, amount: [sum(amount)],
            output=float,
            partition_by=["category", "subcategory"],
        )
        chain.save("new_dataset")
        ```
    """
    if partition_by is not None:
        # Convert string partition_by parameters to Column objects
        if isinstance(partition_by, (str, Function, ColumnElement)):
            list_partition_by = [partition_by]
        else:
            list_partition_by = list(partition_by)

        processed_partition_columns: list[ColumnElement] = []
        for col in list_partition_by:
            if isinstance(col, str):
                columns = self.signals_schema.db_signals(name=col, as_columns=True)
                if not columns:
                    raise SignalResolvingError([col], "is not found")
                processed_partition_columns.extend(cast("list[Column]", columns))
            elif isinstance(col, Function):
                column = col.get_column(self.signals_schema)
                processed_partition_columns.append(column)
            else:
                # Assume it's already a ColumnElement
                processed_partition_columns.append(col)

        processed_partition_by = processed_partition_columns
    else:
        processed_partition_by = []

    udf_obj = self._udf_to_obj(Aggregator, func, params, output, signal_map)
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            partition_by=processed_partition_by,
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

apply 

apply(func, *args, **kwargs)

Apply any function to the chain.

Useful for reusing in a chain of operations.

Example 
import datachain as dc
def parse_stem(chain):
    return chain.map(
        lambda file: file.get_file_stem()
        output={"stem": str}
    )

chain = (
    dc.read_storage("s3://my-bucket")
    .apply(parse_stem)
    .filter(C("stem").glob("*cat*"))
)
Source code in datachain/lib/dc/datachain.py 
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
def apply(self, func, *args, **kwargs):
    """Apply any function to the chain.

    Useful for reusing in a chain of operations.

    Example:
        ```py
        import datachain as dc
        def parse_stem(chain):
            return chain.map(
                lambda file: file.get_file_stem()
                output={"stem": str}
            )

        chain = (
            dc.read_storage("s3://my-bucket")
            .apply(parse_stem)
            .filter(C("stem").glob("*cat*"))
        )
        ```
    """
    return func(self, *args, **kwargs)

avg 

avg(col: str) -> StandardType

Compute the average of a column.

Parameters:

  • col (str) –

    The column to compute the average for.

Returns:

  • StandardType –

    The average of the column values.

Example 
average_size = chain.avg("file.size")
print(f"Average size: {average_size}")
Source code in datachain/lib/dc/datachain.py 
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
def avg(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the average of a column.

    Parameters:
        col: The column to compute the average for.

    Returns:
        The average of the column values.

    Example:
        ```py
        average_size = chain.avg("file.size")
        print(f"Average size: {average_size}")
        ```
    """
    return self._extend_to_data_model("avg", col)

batch_map 

batch_map(
    func: Callable | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map
) -> Self

This is a batch version of map().

Input-output relationship: N:N

It accepts the same parameters plus an additional parameter:

batch: Size of each batch passed to `func`. Defaults to 1000.
Example 
chain = chain.batch_map(
    sqrt=lambda size: np.sqrt(size),
    output=float
)
chain.save("new_dataset")

.. deprecated:: 0.29.0 This method is deprecated and will be removed in a future version. Use agg() instead, which provides the similar functionality.

Source code in datachain/lib/dc/datachain.py 
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
def batch_map(
    self,
    func: Callable | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map,
) -> "Self":
    """This is a batch version of `map()`.

    Input-output relationship: N:N

    It accepts the same parameters plus an
    additional parameter:

        batch: Size of each batch passed to `func`. Defaults to 1000.

    Example:
        ```py
        chain = chain.batch_map(
            sqrt=lambda size: np.sqrt(size),
            output=float
        )
        chain.save("new_dataset")
        ```

    .. deprecated:: 0.29.0
        This method is deprecated and will be removed in a future version.
        Use `agg()` instead, which provides the similar functionality.
    """
    import warnings

    warnings.warn(
        "batch_map() is deprecated and will be removed in a future version. "
        "Use agg() instead, which provides the similar functionality.",
        DeprecationWarning,
        stacklevel=2,
    )
    udf_obj = self._udf_to_obj(BatchMapper, func, params, output, signal_map)

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(self._settings.batch_size, batch=batch),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

c 

c(column: str | Column) -> Column

Returns Column instance attached to the current chain.

Source code in datachain/lib/dc/datachain.py 
278
279
280
281
282
def c(self, column: str | Column) -> Column:
    """Returns Column instance attached to the current chain."""
    c = self.column(column) if isinstance(column, str) else self.column(column.name)
    c.table = self._query.table
    return c

chunk 

chunk(index: int, total: int) -> Self

Split a chain into smaller chunks for e.g. parallelization.

Parameters:

  • index (int) –

    The index of the chunk (0-indexed).

  • total (int) –

    The total number of chunks.

Example 
import datachain as dc

chain = dc.read_storage(...)
chunk_1 = query._chunk(0, 2)
chunk_2 = query._chunk(1, 2)
Note

Bear in mind that index is 0-indexed but total isn't. Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.

Source code in datachain/lib/dc/datachain.py 
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
def chunk(self, index: int, total: int) -> "Self":
    """Split a chain into smaller chunks for e.g. parallelization.

    Parameters:
        index: The index of the chunk (0-indexed).
        total: The total number of chunks.

    Example:
        ```py
        import datachain as dc

        chain = dc.read_storage(...)
        chunk_1 = query._chunk(0, 2)
        chunk_2 = query._chunk(1, 2)
        ```

    Note:
        Bear in mind that `index` is 0-indexed but `total` isn't.
        Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
    """
    return self._evolve(query=self._query.chunk(index, total))

clone 

clone() -> Self

Make a copy of the chain in a new table.

Source code in datachain/lib/dc/datachain.py 
318
319
320
def clone(self) -> "Self":
    """Make a copy of the chain in a new table."""
    return self._evolve(query=self._query.clone(new_table=True))

collect 

collect() -> Iterator[tuple[DataValue, ...]]

collect(col: str) -> Iterator[DataValue]

collect(*cols: str) -> Iterator[tuple[DataValue, ...]]

collect(
    *cols: str,
) -> Iterator[DataValue | tuple[DataValue, ...]]

Deprecated. Use to_iter method instead.

Source code in datachain/lib/dc/datachain.py 
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
def collect(self, *cols: str) -> Iterator[DataValue | tuple[DataValue, ...]]:  # type: ignore[overload-overlap,misc]
    """
    Deprecated. Use `to_iter` method instead.
    """
    warnings.warn(
        "Method `collect` is deprecated. Use `to_iter` method instead.",
        DeprecationWarning,
        stacklevel=2,
    )

    if len(cols) == 1:
        yield from [item[0] for item in self.to_iter(*cols)]
    else:
        yield from self.to_iter(*cols)

column 

column(name: str) -> Column

Returns Column instance with a type if name is found in current schema, otherwise raises an exception.

Source code in datachain/lib/dc/datachain.py 
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
def column(self, name: str) -> Column:
    """Returns Column instance with a type if name is found in current schema,
    otherwise raises an exception.
    """
    if "." in name:
        name_path = name.split(".")
    elif DEFAULT_DELIMITER in name:
        name_path = name.split(DEFAULT_DELIMITER)
    else:
        name_path = [name]
    for path, type_, _, _ in self.signals_schema.get_flat_tree():
        if path == name_path:
            return Column(name, python_to_sql(type_))

    raise ValueError(f"Column with name {name} not found in the schema")

count 

count() -> int

Return the number of rows in the chain.

Source code in datachain/lib/dc/datachain.py 
2858
2859
2860
def count(self) -> int:
    """Return the number of rows in the chain."""
    return self._query.count()

diff 

diff(
    other: DataChain,
    on: str | Sequence[str],
    right_on: str | Sequence[str] | None = None,
    compare: str | Sequence[str] | None = None,
    right_compare: str | Sequence[str] | None = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: str | None = None,
) -> DataChain

Calculate differences between two chains.

This method identifies records that are added, deleted, modified, or unchanged between two chains. It adds a status column with values: A=added, D=deleted, M=modified, S=same.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (str | Sequence[str]) –

    Column(s) to match records between chains.

  • right_on (str | Sequence[str] | None, default: None ) –

    Column(s) in the other chain to match against. Defaults to on.

  • compare (str | Sequence[str] | None, default: None ) –

    Column(s) to check for changes. If not specified,all columns are used.

  • right_compare (str | Sequence[str] | None, default: None ) –

    Column(s) in the other chain to compare against. Defaults to values of compare.

  • added (bool, default: True ) –

    Include records that exist in this chain but not in the other.

  • deleted (bool, default: True ) –

    Include records that exist only in the other chain.

  • modified (bool, default: True ) –

    Include records that exist in both but have different values.

  • same (bool, default: False ) –

    Include records that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences.

Default behavior: By default, shows added, deleted, and modified records, but excludes unchanged records (same=False). Status column is not created.

Example 
res = persons.diff(
    new_persons,
    on=["id"],
    right_on=["other_id"],
    compare=["name"],
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
def diff(
    self,
    other: "DataChain",
    on: str | Sequence[str],
    right_on: str | Sequence[str] | None = None,
    compare: str | Sequence[str] | None = None,
    right_compare: str | Sequence[str] | None = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: str | None = None,
) -> "DataChain":
    """Calculate differences between two chains.

    This method identifies records that are added, deleted, modified, or unchanged
    between two chains. It adds a status column with values: A=added, D=deleted,
    M=modified, S=same.

    Parameters:
        other: Chain to compare against.
        on: Column(s) to match records between chains.
        right_on: Column(s) in the other chain to match against. Defaults to `on`.
        compare: Column(s) to check for changes.
                 If not specified,all columns are used.
        right_compare: Column(s) in the other chain to compare against.
                 Defaults to values of `compare`.
        added (bool): Include records that exist in this chain but not in the other.
        deleted (bool): Include records that exist only in the other chain.
        modified (bool): Include records that exist in both
                 but have different values.
        same (bool): Include records that are identical in both chains.
        status_col (str): Name for the status column showing differences.

    Default behavior: By default, shows added, deleted, and modified records,
    but excludes unchanged records (same=False). Status column is not created.

    Example:
        ```py
        res = persons.diff(
            new_persons,
            on=["id"],
            right_on=["other_id"],
            compare=["name"],
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    from datachain.diff import _compare

    return _compare(
        self,
        other,
        on,
        right_on=right_on,
        compare=compare,
        right_compare=right_compare,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

distinct 

distinct(arg: str, *args: str) -> Self

Removes duplicate rows based on uniqueness of some input column(s) i.e if rows are found with the same value of input column(s), only one row is left in the result set.

Example 
dc.distinct("file.path")
Source code in datachain/lib/dc/datachain.py 
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
@delta_disabled
def distinct(self, arg: str, *args: str) -> "Self":  # type: ignore[override]
    """Removes duplicate rows based on uniqueness of some input column(s)
    i.e if rows are found with the same value of input column(s), only one
    row is left in the result set.

    Example:
        ```py
        dc.distinct("file.path")
        ```
    """
    return self._evolve(
        query=self._query.distinct(
            *self.signals_schema.resolve(arg, *args).db_signals()
        )
    )

exec 

exec() -> Self

Execute the chain.

Source code in datachain/lib/dc/datachain.py 
2862
2863
2864
def exec(self) -> "Self":
    """Execute the chain."""
    return self._evolve(query=self._query.exec())

explode 

explode(
    col: str,
    model_name: str | None = None,
    column: str | None = None,
    schema_sample_size: int = 1,
) -> DataChain

Explodes a column containing JSON objects (dict or str DataChain type) into individual columns based on the schema of the JSON. Schema is inferred from the first row of the column.

Parameters:

  • col (str) –

    the name of the column containing JSON to be exploded.

  • model_name (str | None, default: None ) –

    optional generated model name. By default generates the name automatically.

  • column (str | None, default: None ) –

    optional generated column name. By default generates the name automatically.

  • schema_sample_size (int, default: 1 ) –

    the number of rows to use for inferring the schema of the JSON (in case some fields are optional and it's not enough to analyze a single row).

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance with the new set of columns.

Source code in datachain/lib/dc/datachain.py 
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
def explode(
    self,
    col: str,
    model_name: str | None = None,
    column: str | None = None,
    schema_sample_size: int = 1,
) -> "DataChain":
    """Explodes a column containing JSON objects (dict or str DataChain type) into
       individual columns based on the schema of the JSON. Schema is inferred from
       the first row of the column.

    Args:
        col: the name of the column containing JSON to be exploded.
        model_name: optional generated model name.  By default generates the name
            automatically.
        column: optional generated column name. By default generates the
            name automatically.
        schema_sample_size: the number of rows to use for inferring the schema of
            the JSON (in case some fields are optional and it's not enough to
            analyze a single row).

    Returns:
        DataChain: A new DataChain instance with the new set of columns.
    """
    import pyarrow as pa

    from datachain.lib.arrow import schema_to_output

    json_values = self.limit(schema_sample_size).to_list(col)
    json_dicts = [
        json.loads(json_value) if isinstance(json_value, str) else json_value
        for (json_value,) in json_values
    ]

    if any(not isinstance(json_dict, dict) for json_dict in json_dicts):
        raise TypeError(f"Column {col} should be a string or dict type with JSON")

    schema = pa.Table.from_pylist(json_dicts).schema
    output, original_names = schema_to_output(schema, None)

    if not model_name:
        model_name = f"{col.title()}ExplodedModel"

    model = dict_to_data_model(model_name, output, original_names)

    def json_to_model(json_value: str | dict):
        json_dict = (
            json.loads(json_value) if isinstance(json_value, str) else json_value
        )
        return model.model_validate(json_dict)

    if not column:
        column = f"{col}_expl"

    return self.map(json_to_model, params=col, output={column: model})

file_diff 

file_diff(
    other: DataChain,
    on: str = "file",
    right_on: str | None = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: str | None = None,
) -> DataChain

Calculate differences between two chains containing files.

This method is specifically designed for file chains. It uses file source and path to match files, and file version and etag to detect changes.

Parameters:

  • other (DataChain) –

    Chain to compare against.

  • on (str, default: 'file' ) –

    File column name in this chain. Default is "file".

  • right_on (str | None, default: None ) –

    File column name in the other chain. Defaults to on.

  • added (bool, default: True ) –

    Include files that exist in this chain but not in the other.

  • deleted (bool, default: False ) –

    Include files that exist only in the other chain.

  • modified (bool, default: True ) –

    Include files that exist in both but have different versions/etags.

  • same (bool, default: False ) –

    Include files that are identical in both chains.

  • status_col (str, default: None ) –

    Name for the status column showing differences (A=added, D=deleted, M=modified, S=same).

Default behavior: By default, includes only new files (added=True and modified=True). This is useful for incremental processing.

Example 
diff = images.file_diff(
    new_images,
    on="file",
    right_on="other_file",
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
def file_diff(
    self,
    other: "DataChain",
    on: str = "file",
    right_on: str | None = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: str | None = None,
) -> "DataChain":
    """Calculate differences between two chains containing files.

    This method is specifically designed for file chains. It uses file `source`
    and `path` to match files, and file `version` and `etag` to detect changes.

    Parameters:
        other: Chain to compare against.
        on: File column name in this chain. Default is "file".
        right_on: File column name in the other chain. Defaults to `on`.
        added (bool): Include files that exist in this chain but not in the other.
        deleted (bool): Include files that exist only in the other chain.
        modified (bool): Include files that exist in both but have different
                         versions/etags.
        same (bool): Include files that are identical in both chains.
        status_col (str): Name for the status column showing differences
                          (A=added, D=deleted, M=modified, S=same).

    Default behavior: By default, includes only new files (added=True and
    modified=True). This is useful for incremental processing.

    Example:
        ```py
        diff = images.file_diff(
            new_images,
            on="file",
            right_on="other_file",
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    on_file_signals = ["source", "path"]
    compare_file_signals = ["version", "etag"]

    def get_file_signals(file: str, signals):
        return [f"{file}.{c}" for c in signals]

    right_on = right_on or on

    on_cols = get_file_signals(on, on_file_signals)
    right_on_cols = get_file_signals(right_on, on_file_signals)
    compare_cols = get_file_signals(on, compare_file_signals)
    right_compare_cols = get_file_signals(right_on, compare_file_signals)

    return self.diff(
        other,
        on_cols,
        right_on=right_on_cols,
        compare=compare_cols,
        right_compare=right_compare_cols,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

filter 

filter(*args: Any) -> Self

Filter the chain according to conditions.

Example

Basic usage with built-in operators 

dc.filter(C("width") < 200)

Using glob to match patterns 

dc.filter(C("file.path").glob("*.jpg"))

Using in to match lists 

ids = [1,2,3]
dc.filter(C("experiment_id").in_(ids))

Using datachain.func 

from datachain.func import string
dc.filter(string.length(C("file.path")) > 5)

Combining filters with "or" 

dc.filter(
    C("file.path").glob("cat*") |
    C("file.path").glob("dog*")
)


dc.filter(dc.func.or_(
    C("file.path").glob("cat*"),
    C("file.path").glob("dog*")
))

Combining filters with "and" 

dc.filter(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
)


dc.filter(
    C("file.path").glob("*.jpg") &
    (string.length(C("file.path")) > 5)
)

dc.filter(dc.func.and_(
    C("file.path").glob("*.jpg"),
    string.length(C("file.path")) > 5
))

Combining filters with "not" 

dc.filter(~(C("file.path").glob("*.jpg")))

Source code in datachain/lib/dc/datachain.py 
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
def filter(self, *args: Any) -> "Self":
    """Filter the chain according to conditions.

    Example:
        Basic usage with built-in operators
        ```py
        dc.filter(C("width") < 200)
        ```

        Using glob to match patterns
        ```py
        dc.filter(C("file.path").glob("*.jpg"))
        ```

        Using in to match lists
        ```py
        ids = [1,2,3]
        dc.filter(C("experiment_id").in_(ids))
        ```

        Using `datachain.func`
        ```py
        from datachain.func import string
        dc.filter(string.length(C("file.path")) > 5)
        ```

        Combining filters with "or"
        ```py
        dc.filter(
            C("file.path").glob("cat*") |
            C("file.path").glob("dog*")
        )
        ```

        ```py
        dc.filter(dc.func.or_(
            C("file.path").glob("cat*"),
            C("file.path").glob("dog*")
        ))
        ```

        Combining filters with "and"
        ```py
        dc.filter(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        )
        ```

        ```py
        dc.filter(
            C("file.path").glob("*.jpg") &
            (string.length(C("file.path")) > 5)
        )
        ```

        ```py
        dc.filter(dc.func.and_(
            C("file.path").glob("*.jpg"),
            string.length(C("file.path")) > 5
        ))
        ```

        Combining filters with "not"
        ```py
        dc.filter(~(C("file.path").glob("*.jpg")))
        ```
    """
    return self._evolve(query=self._query.filter(*args))

gen 

gen(
    func: Callable | Generator | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new rows (with potentially new signals). The function needs to return a new objects for each of the new rows. It returns a chain itself with new signals.

Input-output relationship: 1:N

This method is similar to map(), uses the same list of parameters, but with one key differences: It produces a sequence of rows for each input row (like extracting multiple file records from a single tar file or bounding boxes from a single image file).

Example 
chain = chain.gen(
    line=lambda file: [l for l in file.read().split("\n")],
    output=str,
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
def gen(
    self,
    func: Callable | Generator | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    r"""Apply a function to each row to create new rows (with potentially new
    signals). The function needs to return a new objects for each of the new rows.
    It returns a chain itself with new signals.

    Input-output relationship: 1:N

    This method is similar to `map()`, uses the same list of parameters, but with
    one key differences: It produces a sequence of rows for each input row (like
    extracting multiple file records from a single tar file or bounding boxes from a
    single image file).

    Example:
        ```py
        chain = chain.gen(
            line=lambda file: [l for l in file.read().split("\n")],
            output=str,
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Generator, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

group_by 

group_by(
    *,
    partition_by: (
        str | Func | Sequence[str | Func] | None
    ) = None,
    **kwargs: Func
) -> Self

Group rows by specified set of signals and return new signals with aggregated values.

The supported functions

count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

Example 
chain = chain.group_by(
    cnt=func.count(),
    partition_by=("file_source", "file_ext"),
)

Using complex signals: 

chain = chain.group_by(
    total_size=func.sum("file.size"),
    count=func.count(),
    partition_by="file",  # Uses column name, expands to File's unique keys
)

Source code in datachain/lib/dc/datachain.py 
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
@delta_disabled  # type: ignore[arg-type]
def group_by(  # noqa: C901, PLR0912
    self,
    *,
    partition_by: str | Func | Sequence[str | Func] | None = None,
    **kwargs: Func,
) -> "Self":
    """Group rows by specified set of signals and return new signals
    with aggregated values.

    The supported functions:
       count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

    Example:
        ```py
        chain = chain.group_by(
            cnt=func.count(),
            partition_by=("file_source", "file_ext"),
        )
        ```

        Using complex signals:
        ```py
        chain = chain.group_by(
            total_size=func.sum("file.size"),
            count=func.count(),
            partition_by="file",  # Uses column name, expands to File's unique keys
        )
        ```
    """
    if partition_by is None:
        partition_by = []
    elif isinstance(partition_by, (str, Func)):
        partition_by = [partition_by]

    partition_by_columns: list[Column] = []
    signal_columns: list[Column] = []
    schema_fields: dict[str, DataType] = {}
    keep_columns: list[str] = []
    partial_fields: list[str] = []  # Track specific fields for partial creation
    schema_partition_by: list[str] = []

    for col in partition_by:
        if isinstance(col, str):
            columns = self.signals_schema.db_signals(name=col, as_columns=True)
            if not columns:
                raise SignalResolvingError([col], "is not found")
            partition_by_columns.extend(cast("list[Column]", columns))

            # For nested field references (e.g., "nested.level1.name"),
            # we need to distinguish between:
            # 1. References to fields within a complex signal (create partials)
            # 2. Deep nested references that should be flattened
            if "." in col:
                # Split the column reference to analyze it
                parts = col.split(".")
                parent_signal = parts[0]
                parent_type = self.signals_schema.values.get(parent_signal)

                if ModelStore.is_partial(parent_type):
                    if parent_signal not in keep_columns:
                        keep_columns.append(parent_signal)
                    partial_fields.append(col)
                    schema_partition_by.append(col)
                else:
                    # BaseModel or other - add flattened columns directly
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
            else:
                # simple signal - but we need to check if it's a complex signal
                # complex signal - only include the columns used for partitioning
                col_type = self.signals_schema.get_column_type(
                    col, with_subtree=True
                )
                if isinstance(col_type, type) and issubclass(col_type, BaseModel):
                    # Complex signal - add only the partitioning columns
                    for column in cast("list[Column]", columns):
                        col_type = self.signals_schema.get_column_type(column.name)
                        schema_fields[column.name] = col_type
                    schema_partition_by.append(col)
                # Simple signal - keep the entire signal
                else:
                    if col not in keep_columns:
                        keep_columns.append(col)
                    schema_partition_by.append(col)
        elif isinstance(col, Function):
            column = col.get_column(self.signals_schema)
            col_db_name = column.name
            col_type = column.type.python_type
            schema_fields[col_db_name] = col_type
            partition_by_columns.append(column)
            signal_columns.append(column)
        else:
            raise DataChainColumnError(
                col,
                (
                    f"partition_by column {col} has type {type(col)}"
                    " but expected str or Function"
                ),
            )

    if not kwargs:
        raise ValueError("At least one column should be provided for group_by")
    for col_name, func in kwargs.items():
        if not isinstance(func, Func):
            raise DataChainColumnError(
                col_name,
                f"Column {col_name} has type {type(func)} but expected Func object",
            )
        column = func.get_column(self.signals_schema, label=col_name)
        signal_columns.append(column)
        schema_fields[col_name] = func.get_result_type(self.signals_schema)

    signal_schema = self.signals_schema.group_by(
        schema_partition_by, signal_columns
    )

    return self._evolve(
        query=self._query.group_by(signal_columns, partition_by_columns),
        signal_schema=signal_schema,
    )

hash 

hash() -> str

Calculates SHA hash of this chain. Hash calculation is fast and consistent. It takes into account all the steps added to the chain and their inputs. Order of the steps is important.

Source code in datachain/lib/dc/datachain.py 
215
216
217
218
219
220
221
def hash(self) -> str:
    """
    Calculates SHA hash of this chain. Hash calculation is fast and consistent.
    It takes into account all the steps added to the chain and their inputs.
    Order of the steps is important.
    """
    return self._query.hash()

limit 

limit(n: int) -> Self

Return the first n rows of the chain.

If the chain is unordered, which rows are returned is undefined. If the chain has less than n rows, the whole chain is returned.

Parameters:

  • n (int) –

    Number of rows to return.

Source code in datachain/lib/dc/datachain.py 
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
def limit(self, n: int) -> "Self":
    """Return the first `n` rows of the chain.

    If the chain is unordered, which rows are returned is undefined.
    If the chain has less than `n` rows, the whole chain is returned.

    Parameters:
        n (int): Number of rows to return.
    """
    return self._evolve(query=self._query.limit(n))

map 

map(
    func: Callable | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map: Any
) -> Self

Apply a function to each row to create new signals. The function should return a new object for each row. It returns a chain itself with new signals.

Input-output relationship: 1:1

Parameters:

  • func (Callable | None, default: None ) –

    Function applied to each row.

  • params (str | Sequence[str] | None, default: None ) –

    List of column names used as input for the function. Default is taken from function signature.

  • output (OutputType, default: None ) –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature. Default can be also taken from kwargs - **signal_map (see below). If signal name is defined using signal_map (see below) only a single type value can be used.

  • **signal_map (Any, default: {} ) –

    kwargs can be used to define func together with its return signal name in format of map(my_sign=my_func). This helps define signal names and functions in a nicer way.

Example

Using signal_map and single type in output: 

chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
chain.save("new_dataset")

Using func and output as a map: 

chain = chain.map(
    lambda name: name.split("."), output={"stem": str, "ext": str}
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
def map(
    self,
    func: Callable | None = None,
    params: str | Sequence[str] | None = None,
    output: OutputType = None,
    **signal_map: Any,
) -> "Self":
    """Apply a function to each row to create new signals. The function should
    return a new object for each row. It returns a chain itself with new signals.

    Input-output relationship: 1:1

    Parameters:
        func: Function applied to each row.
        params: List of column names used as input for the function. Default
                is taken from function signature.
        output: Dictionary defining new signals and their corresponding types.
                Default type is taken from function signature. Default can be also
                taken from kwargs - **signal_map (see below).
                If signal name is defined using signal_map (see below) only a single
                type value can be used.
        **signal_map: kwargs can be used to define `func` together with its return
                signal name in format of `map(my_sign=my_func)`. This helps define
                signal names and functions in a nicer way.

    Example:
        Using signal_map and single type in output:
        ```py
        chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
        chain.save("new_dataset")
        ```

        Using func and output as a map:
        ```py
        chain = chain.map(
            lambda name: name.split("."), output={"stem": str, "ext": str}
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Mapper, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(self._settings.batch_size),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

max 

max(col: str) -> StandardType

Compute the maximum of a column.

Parameters:

  • col (str) –

    The column to compute the maximum for.

Returns:

  • StandardType –

    The maximum value in the column.

Example 
max_size = chain.max("file.size")
print(f"Maximum size: {max_size}")
Source code in datachain/lib/dc/datachain.py 
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
def max(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the maximum of a column.

    Parameters:
        col: The column to compute the maximum for.

    Returns:
        The maximum value in the column.

    Example:
        ```py
        max_size = chain.max("file.size")
        print(f"Maximum size: {max_size}")
        ```
    """
    return self._extend_to_data_model("max", col)

merge 

merge(
    right_ds: DataChain,
    on: MergeColType | Sequence[MergeColType],
    right_on: (
        MergeColType | Sequence[MergeColType] | None
    ) = None,
    inner=False,
    full=False,
    rname="right_",
) -> Self

Merge two chains based on the specified criteria.

Parameters:

  • right_ds (DataChain) –

    Chain to join with.

  • on (MergeColType | Sequence[MergeColType]) –

    Predicate ("column.name", C("column.name"), or Func) or list of Predicates to join on. If both chains have the same predicates then this predicate is enough for the join. Otherwise, right_on parameter has to specify the predicates for the other chain.

  • right_on (MergeColType | Sequence[MergeColType] | None, default: None ) –

    Optional predicate or list of Predicates for the right_ds to join.

  • inner (bool, default: False ) –

    Whether to run inner join or outer join.

  • full (bool, default: False ) –

    Whether to run full outer join.

  • rname (str, default: 'right_' ) –

    Name prefix for conflicting signal names.

Examples:

meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                      right_on=(C.name, C.pq__index))

imgs.merge(captions,
           on=func.path.file_stem(imgs.c("file.path")),
           right_on=func.path.file_stem(captions.c("file.path"))

)

Source code in datachain/lib/dc/datachain.py 
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
@delta_disabled
def merge(
    self,
    right_ds: "DataChain",
    on: MergeColType | Sequence[MergeColType],
    right_on: MergeColType | Sequence[MergeColType] | None = None,
    inner=False,
    full=False,
    rname="right_",
) -> "Self":
    """Merge two chains based on the specified criteria.

    Parameters:
        right_ds: Chain to join with.
        on: Predicate ("column.name", C("column.name"), or Func) or list of
            Predicates to join on. If both chains have the same predicates then
            this predicate is enough for the join. Otherwise, `right_on` parameter
            has to specify the predicates for the other chain.
        right_on: Optional predicate or list of Predicates for the `right_ds`
            to join.
        inner (bool): Whether to run inner join or outer join.
        full (bool): Whether to run full outer join.
        rname (str): Name prefix for conflicting signal names.

    Examples:
        ```py
        meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                              right_on=(C.name, C.pq__index))
        ```

        ```py
        imgs.merge(captions,
                   on=func.path.file_stem(imgs.c("file.path")),
                   right_on=func.path.file_stem(captions.c("file.path"))
        ```
    )
    """
    if on is None:
        raise DatasetMergeError(["None"], None, "'on' must be specified")

    on = _validate_merge_on(on, self)
    if not on:
        raise DatasetMergeError(
            on,
            right_on,
            (
                "'on' must be 'str', 'Func' or 'Sequence' object "
                f"but got type '{type(on)}'"
            ),
        )

    if right_on is not None:
        right_on = _validate_merge_on(right_on, right_ds)
        if not right_on:
            raise DatasetMergeError(
                on,
                right_on,
                "'right_on' must be 'str', 'Func' or 'Sequence' object"
                f" but got type '{type(right_on)}'",
            )

        if len(right_on) != len(on):
            raise DatasetMergeError(
                on, right_on, "'on' and 'right_on' must have the same length'"
            )

    if self == right_ds:
        right_ds = right_ds.clone()

    errors = []

    def _resolve(
        ds: DataChain,
        col: str | Function | sqlalchemy.ColumnElement,
        side: str | None,
    ):
        try:
            if isinstance(col, Function):
                return ds.c(col.get_column())
            return ds.c(col) if isinstance(col, (str, C)) else col
        except ValueError:
            if side:
                errors.append(f"{_get_merge_error_str(col)} in {side}")

    ops = [
        _resolve(self, left, "left")
        == _resolve(right_ds, right, "right" if right_on else None)
        for left, right in zip(on, right_on or on, strict=False)
    ]

    if errors:
        raise DatasetMergeError(
            on, right_on, f"Could not resolve {', '.join(errors)}"
        )

    query = self._query.join(
        right_ds._query,
        sqlalchemy.and_(*ops),
        inner,
        full,
        rname + "{name}",
    )
    query.feature_schema = None
    ds = self._evolve(query=query)

    signals_schema = self.signals_schema.clone_without_sys_signals()
    right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
    ds.signals_schema = SignalSchema({"sys": Sys}) | signals_schema.merge(
        right_signals_schema, rname
    )

    return ds

min 

min(col: str) -> StandardType

Compute the minimum of a column.

Parameters:

  • col (str) –

    The column to compute the minimum for.

Returns:

  • StandardType –

    The minimum value in the column.

Example 
min_size = chain.min("file.size")
print(f"Minimum size: {min_size}")
Source code in datachain/lib/dc/datachain.py 
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
def min(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the minimum of a column.

    Parameters:
        col: The column to compute the minimum for.

    Returns:
        The minimum value in the column.

    Example:
        ```py
        min_size = chain.min("file.size")
        print(f"Minimum size: {min_size}")
        ```
    """
    return self._extend_to_data_model("min", col)

mutate 

mutate(**kwargs) -> Self

Create or modify signals based on existing signals.

This method is vectorized and more efficient compared to map(), and it does not extract or download any data from the internal database. However, it can only utilize predefined built-in functions and their combinations.

Supported functions

Numerical: +, -, *, /, rand(), avg(), count(), func(), greatest(), least(), max(), min(), sum() String: length(), split(), replace(), regexp_replace() Filename: name(), parent(), file_stem(), file_ext() Array: length(), sip_hash_64(), euclidean_distance(), cosine_distance() Window: row_number(), rank(), dense_rank(), first()

Example: 

 dc.mutate(
    area=Column("image.height") * Column("image.width"),
    extension=file_ext(Column("file.path")),
    dist=cosine_distance(embedding_text, embedding_image)
)

Window function example: 

window = func.window(partition_by="file.parent", order_by="file.size")
dc.mutate(
    row_number=func.row_number().over(window),
)

This method can be also used to rename signals. If the Column("name") provided as value for the new signal - the old signal will be dropped. Otherwise a new signal is created. Exception, if the old signal is nested one (e.g. C("file.path")), it will be kept to keep the object intact.

Example: 

 dc.mutate(
    newkey=Column("oldkey") # drops oldkey
)


 dc.mutate(
    size=Column("file.size") # keeps `file.size`
)
Source code in datachain/lib/dc/datachain.py 
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
def mutate(self, **kwargs) -> "Self":
    """Create or modify signals based on existing signals.

    This method is vectorized and more efficient compared to map(), and it does not
    extract or download any data from the internal database. However, it can only
    utilize predefined built-in functions and their combinations.

    Supported functions:
       Numerical:   +, -, *, /, rand(), avg(), count(), func(),
                    greatest(), least(), max(), min(), sum()
       String:      length(), split(), replace(), regexp_replace()
       Filename:    name(), parent(), file_stem(), file_ext()
       Array:       length(), sip_hash_64(), euclidean_distance(),
                    cosine_distance()
       Window:      row_number(), rank(), dense_rank(), first()

    Example:
    ```py
     dc.mutate(
        area=Column("image.height") * Column("image.width"),
        extension=file_ext(Column("file.path")),
        dist=cosine_distance(embedding_text, embedding_image)
    )
    ```

    Window function example:
    ```py
    window = func.window(partition_by="file.parent", order_by="file.size")
    dc.mutate(
        row_number=func.row_number().over(window),
    )
    ```

    This method can be also used to rename signals. If the Column("name") provided
    as value for the new signal - the old signal will be dropped. Otherwise a new
    signal is created. Exception, if the old signal is nested one (e.g.
    `C("file.path")`), it will be kept to keep the object intact.

    Example:
    ```py
     dc.mutate(
        newkey=Column("oldkey") # drops oldkey
    )
    ```

    ```py
     dc.mutate(
        size=Column("file.size") # keeps `file.size`
    )
    ```
    """
    from sqlalchemy.sql.sqltypes import NullType

    primitives = (bool, str, int, float)

    for col_name, expr in kwargs.items():
        if not isinstance(expr, (*primitives, Column, Func)) and isinstance(
            expr.type, NullType
        ):
            raise DataChainColumnError(
                col_name, f"Cannot infer type with expression {expr}"
            )

    mutated = {}
    schema = self.signals_schema
    for name, value in kwargs.items():
        if isinstance(value, Column):
            # renaming existing column
            for signal in schema.db_signals(name=value.name, as_columns=True):
                mutated[signal.name.replace(value.name, name, 1)] = signal  # type: ignore[union-attr]
        elif isinstance(value, Func):
            # adding new signal
            mutated[name] = value.get_column(schema)
        elif isinstance(value, primitives):
            # adding simple python constant primitives like str, int, float, bool
            val = literal(value)
            val.type = python_to_sql(type(value))()
            mutated[name] = val  # type: ignore[assignment]
        else:
            # adding new signal
            mutated[name] = value

    new_schema = schema.mutate(kwargs)
    return self._evolve(
        query=self._query.mutate(new_schema=new_schema, **mutated),
        signal_schema=new_schema,
    )

offset 

offset(offset: int) -> Self

Return the results starting with the offset row.

If the chain is unordered, which rows are skipped in undefined. If the chain has less than offset rows, the result is an empty chain.

Parameters:

  • offset (int) –

    Number of rows to skip.

Source code in datachain/lib/dc/datachain.py 
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
def offset(self, offset: int) -> "Self":
    """Return the results starting with the offset row.

    If the chain is unordered, which rows are skipped in undefined.
    If the chain has less than `offset` rows, the result is an empty chain.

    Parameters:
        offset (int): Number of rows to skip.
    """
    return self._evolve(query=self._query.offset(offset))

order_by 

order_by(*args, descending: bool = False) -> Self

Orders by specified set of columns.

Parameters:

  • descending (bool, default: False ) –

    Whether to sort in descending order or not.

Example 
dc.order_by("similarity_score", descending=True).limit(10)
Note

Order is not guaranteed when steps are added after an order_by statement. I.e. when using read_dataset an order_by statement should be used if the order of the records in the chain is important. Using order_by directly before limit, to_list and similar methods will give expected results. See https://github.com/iterative/datachain/issues/477 for further details.

Source code in datachain/lib/dc/datachain.py 
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
@resolve_columns
def order_by(self, *args, descending: bool = False) -> "Self":
    """Orders by specified set of columns.

    Parameters:
        descending (bool): Whether to sort in descending order or not.

    Example:
        ```py
        dc.order_by("similarity_score", descending=True).limit(10)
        ```

    Note:
        Order is not guaranteed when steps are added after an `order_by` statement.
        I.e. when using `read_dataset` an `order_by` statement should be used if
        the order of the records in the chain is important.
        Using `order_by` directly before `limit`, `to_list` and similar methods
        will give expected results.
        See https://github.com/iterative/datachain/issues/477 for further details.
    """
    if descending:
        args = tuple(sqlalchemy.desc(a) for a in args)

    return self._evolve(query=self._query.order_by(*args))

parse_tabular 

parse_tabular(
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: int | None = None,
    **kwargs: Any
) -> Self

Generate chain from list of tabular files.

Parameters:

  • output (OutputType, default: None ) –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • column (str, default: '' ) –

    Generated column name.

  • model_name (str, default: '' ) –

    Generated model name.

  • source (bool, default: True ) –

    Whether to include info about the source file.

  • nrows (int | None, default: None ) –

    Optional row limit.

  • kwargs (Any, default: {} ) –

    Parameters to pass to pyarrow.dataset.dataset.

Example

Reading a json lines file: 

import datachain as dc
chain = dc.read_storage("s3://mybucket/file.jsonl")
chain = chain.parse_tabular(format="json")

Reading a filtered list of files as a dataset: 

import datachain as dc

chain = dc.read_storage("s3://mybucket")
chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
chain = chain.parse_tabular(format="json")

Source code in datachain/lib/dc/datachain.py 
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
def parse_tabular(
    self,
    output: OutputType = None,
    column: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: int | None = None,
    **kwargs: Any,
) -> "Self":
    """Generate chain from list of tabular files.

    Parameters:
        output: Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        column: Generated column name.
        model_name: Generated model name.
        source: Whether to include info about the source file.
        nrows: Optional row limit.
        kwargs: Parameters to pass to pyarrow.dataset.dataset.

    Example:
        Reading a json lines file:
        ```py
        import datachain as dc
        chain = dc.read_storage("s3://mybucket/file.jsonl")
        chain = chain.parse_tabular(format="json")
        ```

        Reading a filtered list of files as a dataset:
        ```py
        import datachain as dc

        chain = dc.read_storage("s3://mybucket")
        chain = chain.filter(dc.C("file.path").glob("*.jsonl"))
        chain = chain.parse_tabular(format="json")
        ```
    """
    from pyarrow.dataset import CsvFileFormat, JsonFileFormat

    from datachain.lib.arrow import (
        ArrowGenerator,
        fix_pyarrow_format,
        infer_schema,
        schema_to_output,
    )

    parse_options = kwargs.pop("parse_options", None)
    if format := kwargs.get("format"):
        kwargs["format"] = fix_pyarrow_format(format, parse_options)

    if (
        nrows
        and format not in ["csv", "json"]
        and not isinstance(format, (CsvFileFormat, JsonFileFormat))
    ):
        raise DatasetPrepareError(
            self.name,
            "error in `parse_tabular` - "
            "`nrows` only supported for csv and json formats.",
        )

    if "file" not in self.schema or not self.count():
        raise DatasetPrepareError(self.name, "no files to parse.")

    schema = None
    col_names = output if isinstance(output, Sequence) else None
    if col_names or not output:
        try:
            schema = infer_schema(self, **kwargs, parse_options=parse_options)
            output, _ = schema_to_output(schema, col_names)
        except ValueError as e:
            raise DatasetPrepareError(self.name, e) from e

    if isinstance(output, dict):
        model_name = model_name or column or ""
        model = dict_to_data_model(model_name, output)
        output = model
    else:
        model = output  # type: ignore[assignment]

    if column:
        output = {column: model}  # type: ignore[dict-item]
    elif isinstance(output, type(BaseModel)):
        output = {
            name: info.annotation  # type: ignore[misc]
            for name, info in output.model_fields.items()
        }

    if source:
        output = {"source": ArrowRow} | output  # type: ignore[assignment,operator]

    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}
    return self.settings(**settings).gen(  # type: ignore[arg-type]
        ArrowGenerator(
            schema,
            model,
            source,
            nrows,
            parse_options=parse_options,
            **kwargs,
        ),
        output=output,
    )

persist 

persist() -> Self

Saves temporary chain that will be removed after the process ends. Temporary datasets are useful for optimization, for example when we have multiple chains starting with identical sub-chain. We can then persist that common chain and use it to calculate other chains, to avoid re-calculation every time. It returns the chain itself.

Source code in datachain/lib/dc/datachain.py 
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
def persist(self) -> "Self":
    """Saves temporary chain that will be removed after the process ends.
    Temporary datasets are useful for optimization, for example when we have
    multiple chains starting with identical sub-chain. We can then persist that
    common chain and use it to calculate other chains, to avoid re-calculation
    every time.
    It returns the chain itself.
    """
    schema = self.signals_schema.clone_without_sys_signals().serialize()
    project = self.session.catalog.metastore.get_project(
        self.project_name,
        self.namespace_name,
        create=True,
    )
    return self._evolve(
        query=self._query.save(project=project, feature_schema=schema)
    )

print_schema 

print_schema(file: IO | None = None) -> None

Print schema of the chain.

Source code in datachain/lib/dc/datachain.py 
314
315
316
def print_schema(self, file: IO | None = None) -> None:
    """Print schema of the chain."""
    self._effective_signals_schema.print_tree(file=file)

reset_settings 

reset_settings(settings: Settings | None = None) -> Self

Reset all chain settings to default values.

Source code in datachain/lib/dc/datachain.py 
413
414
415
416
def reset_settings(self, settings: Settings | None = None) -> "Self":
    """Reset all chain settings to default values."""
    self._settings = settings if settings else Settings()
    return self

sample 

sample(n: int) -> Self

Return a random sample from the chain.

Parameters:

  • n (int) –

    Number of samples to draw.

Note

Samples are not deterministic, and streamed/paginated queries or multiple workers will draw samples with replacement.

Source code in datachain/lib/dc/datachain.py 
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
def sample(self, n: int) -> "Self":
    """Return a random sample from the chain.

    Parameters:
        n: Number of samples to draw.

    Note:
        Samples are not deterministic, and streamed/paginated queries or
        multiple workers will draw samples with replacement.
    """
    return self._evolve(query=self._query.sample(n))

save 

save(
    name: str,
    version: str | None = None,
    description: str | None = None,
    attrs: list[str] | None = None,
    update_version: str | None = "patch",
    **kwargs
) -> DataChain

Save to a Dataset. It returns the chain itself.

Parameters:

  • name (str) –

    dataset name. This can be either a fully qualified name, including the namespace and project, or just a regular dataset name. In the latter case, the namespace and project will be taken from the settings (if specified) or from the default values otherwise.

  • version (str | None, default: None ) –

    version of a dataset. If version is not specified and dataset already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.

  • description (str | None, default: None ) –

    description of a dataset.

  • attrs (list[str] | None, default: None ) –

    attributes of a dataset. They can be without value, e.g "NLP", or with a value, e.g "location=US".

  • update_version (str | None, default: 'patch' ) –

    which part of the dataset version to automatically increase. Available values: major, minor or patch. Default is patch.

Source code in datachain/lib/dc/datachain.py 
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
def save(  # type: ignore[override]
    self,
    name: str,
    version: str | None = None,
    description: str | None = None,
    attrs: list[str] | None = None,
    update_version: str | None = "patch",
    **kwargs,
) -> "DataChain":
    """Save to a Dataset. It returns the chain itself.

    Parameters:
        name: dataset name. This can be either a fully qualified name, including
            the namespace and project, or just a regular dataset name. In the latter
            case, the namespace and project will be taken from the settings
            (if specified) or from the default values otherwise.
        version: version of a dataset. If version is not specified and dataset
            already exists, version patch increment will happen e.g 1.2.1 -> 1.2.2.
        description: description of a dataset.
        attrs: attributes of a dataset. They can be without value, e.g "NLP",
            or with a value, e.g "location=US".
        update_version: which part of the dataset version to automatically increase.
            Available values: `major`, `minor` or `patch`. Default is `patch`.
    """

    catalog = self.session.catalog

    result = None  # result chain that will be returned at the end

    # Version validation
    self._validate_version(version)
    self._validate_update_version(update_version)

    namespace_name, project_name, name = catalog.get_full_dataset_name(
        name,
        namespace_name=self._settings.namespace,
        project_name=self._settings.project,
    )
    project = self._get_or_create_project(namespace_name, project_name)

    # Checkpoint handling
    job, _hash, result = self._resolve_checkpoint(name, project, kwargs)

    # Schema preparation
    schema = self.signals_schema.clone_without_sys_signals().serialize()

    # Handle retry and delta functionality
    if not result:
        result = self._handle_delta(name, version, project, schema, kwargs)

    if not result:
        # calculate chain if we already don't have result from checkpoint or delta
        result = self._evolve(
            query=self._query.save(
                name=name,
                version=version,
                project=project,
                description=description,
                attrs=attrs,
                feature_schema=schema,
                update_version=update_version,
                **kwargs,
            )
        )

    if job:
        catalog.metastore.create_checkpoint(job.id, _hash)  # type: ignore[arg-type]

    return result

select 

select(*args: str, _sys: bool = True) -> Self

Select only a specified set of signals.

Source code in datachain/lib/dc/datachain.py 
1159
1160
1161
1162
1163
1164
1165
1166
1167
def select(self, *args: str, _sys: bool = True) -> "Self":
    """Select only a specified set of signals."""
    new_schema = self.signals_schema.resolve(*args)
    if self._sys and _sys:
        new_schema = SignalSchema({"sys": Sys}) | new_schema
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

select_except 

select_except(*args: str) -> Self

Select all the signals expect the specified signals.

Source code in datachain/lib/dc/datachain.py 
1169
1170
1171
1172
1173
1174
1175
def select_except(self, *args: str) -> "Self":
    """Select all the signals expect the specified signals."""
    new_schema = self.signals_schema.select_except_signals(*args)
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

settings 

settings(
    cache: bool | None = None,
    prefetch: bool | int | None = None,
    parallel: bool | int | None = None,
    workers: int | None = None,
    namespace: str | None = None,
    project: str | None = None,
    min_task_size: int | None = None,
    batch_size: int | None = None,
    sys: bool | None = None,
) -> Self

Set chain execution parameters. Returns the chain itself, allowing method chaining for subsequent operations. To restore all settings to their default values, use reset_settings().

Parameters:

  • cache (bool | None, default: None ) –

    Enable files caching to speed up subsequent accesses to the same files from the same or different chains. Defaults to False.

  • prefetch (bool | int | None, default: None ) –

    Enable prefetching of files. This will download files in advance in parallel. If an integer is provided, it specifies the number of files to prefetch concurrently for each process on each worker. Defaults to 2. Set to 0 or False to disable prefetching.

  • parallel (bool | int | None, default: None ) –

    Number of processes to use for processing user-defined functions (UDFs) in parallel. If an integer is provided, it specifies the number of CPUs to use. If True, all available CPUs are used. Defaults to 1.

  • namespace (str | None, default: None ) –

    Namespace to use for the chain by default.

  • project (str | None, default: None ) –

    Project to use for the chain by default.

  • min_task_size (int | None, default: None ) –

    Minimum number of rows per worker/process for parallel processing by UDFs. Defaults to 1.

  • batch_size (int | None, default: None ) –

    Number of rows per insert by UDF to fine tune and balance speed and memory usage. This might be useful when processing large rows or when running into memory issues. Defaults to 2000.

Example 
chain = (
    chain
    .settings(cache=True, parallel=8, batch_size=300)
    .map(laion=process_webdataset(spec=WDSLaion), params="file")
)
Source code in datachain/lib/dc/datachain.py 
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
def settings(
    self,
    cache: bool | None = None,
    prefetch: bool | int | None = None,
    parallel: bool | int | None = None,
    workers: int | None = None,
    namespace: str | None = None,
    project: str | None = None,
    min_task_size: int | None = None,
    batch_size: int | None = None,
    sys: bool | None = None,
) -> "Self":
    """
    Set chain execution parameters. Returns the chain itself, allowing method
    chaining for subsequent operations. To restore all settings to their default
    values, use `reset_settings()`.

    Parameters:
        cache: Enable files caching to speed up subsequent accesses to the same
            files from the same or different chains. Defaults to False.
        prefetch: Enable prefetching of files. This will download files in
            advance in parallel. If an integer is provided, it specifies the number
            of files to prefetch concurrently for each process on each worker.
            Defaults to 2. Set to 0 or False to disable prefetching.
        parallel: Number of processes to use for processing user-defined functions
            (UDFs) in parallel. If an integer is provided, it specifies the number
            of CPUs to use. If True, all available CPUs are used. Defaults to 1.
        namespace: Namespace to use for the chain by default.
        project: Project to use for the chain by default.
        min_task_size: Minimum number of rows per worker/process for parallel
            processing by UDFs. Defaults to 1.
        batch_size: Number of rows per insert by UDF to fine tune and balance speed
            and memory usage. This might be useful when processing large rows
            or when running into memory issues. Defaults to 2000.

    Example:
        ```py
        chain = (
            chain
            .settings(cache=True, parallel=8, batch_size=300)
            .map(laion=process_webdataset(spec=WDSLaion), params="file")
        )
        ```
    """
    if sys is None:
        sys = self._sys
    settings = copy.copy(self._settings)
    settings.add(
        Settings(
            cache=cache,
            prefetch=prefetch,
            parallel=parallel,
            workers=workers,
            namespace=namespace,
            project=project,
            min_task_size=min_task_size,
            batch_size=batch_size,
        )
    )
    return self._evolve(settings=settings, _sys=sys)

setup 

setup(**kwargs) -> Self

Setup variables to pass to UDF functions.

Use before running map/gen/agg to save an object and pass it as an argument to the UDF.

The value must be a callable (a lambda: <value> syntax can be used to quickly create one) that returns the object to be passed to the UDF. It is evaluated lazily when UDF is running, in case of multiple machines the callable is run on a worker machine.

Example 
import anthropic
from anthropic.types import Message
import datachain as dc

(
    dc.read_storage(DATA, type="text")
    .settings(parallel=4, cache=True)

    # Setup Anthropic client and pass it to the UDF below automatically
    # The value is callable (see the note above)
    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

    .map(
        claude=lambda client, file: client.messages.create(
            model=MODEL,
            system=PROMPT,
            messages=[{"role": "user", "content": file.get_value()}],
        ),
        output=Message,
    )
)
Source code in datachain/lib/dc/datachain.py 
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
def setup(self, **kwargs) -> "Self":
    """Setup variables to pass to UDF functions.

    Use before running map/gen/agg to save an object and pass it as an
    argument to the UDF.

    The value must be a callable (a `lambda: <value>` syntax can be used to quickly
    create one) that returns the object to be passed to the UDF. It is evaluated
    lazily when UDF is running, in case of multiple machines the callable is run on
    a worker machine.

    Example:
        ```py
        import anthropic
        from anthropic.types import Message
        import datachain as dc

        (
            dc.read_storage(DATA, type="text")
            .settings(parallel=4, cache=True)

            # Setup Anthropic client and pass it to the UDF below automatically
            # The value is callable (see the note above)
            .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))

            .map(
                claude=lambda client, file: client.messages.create(
                    model=MODEL,
                    system=PROMPT,
                    messages=[{"role": "user", "content": file.get_value()}],
                ),
                output=Message,
            )
        )
        ```
    """
    intersection = set(self._setup.keys()) & set(kwargs.keys())
    if intersection:
        keys = ", ".join(intersection)
        raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")

    self._setup = self._setup | kwargs
    return self

show 

show(
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None

Show a preview of the chain results.

Parameters:

  • limit (int, default: 20 ) –

    How many rows to show.

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • transpose (bool, default: False ) –

    Whether to transpose rows and columns.

  • truncate (bool, default: True ) –

    Whether or not to truncate the contents of columns.

  • include_hidden (bool, default: False ) –

    Whether to include hidden columns.

Source code in datachain/lib/dc/datachain.py 
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
def show(
    self,
    limit: int = 20,
    flatten: bool = False,
    transpose: bool = False,
    truncate: bool = True,
    include_hidden: bool = False,
) -> None:
    """Show a preview of the chain results.

    Parameters:
        limit: How many rows to show.
        flatten: Whether to use a multiindex or flatten column names.
        transpose: Whether to transpose rows and columns.
        truncate: Whether or not to truncate the contents of columns.
        include_hidden: Whether to include hidden columns.
    """
    import pandas as pd

    dc = self.limit(limit) if limit > 0 else self  # type: ignore[misc]
    df = dc.to_pandas(
        flatten,
        include_hidden=include_hidden,
        as_object=True,
    )

    if df.empty:
        print("Empty result")
        print(f"Columns: {list(df.columns)}")
        return

    if transpose:
        df = df.T

    options: list = [
        "display.max_columns",
        None,
        "display.multi_sparse",
        False,
    ]

    try:
        if columns := os.get_terminal_size().columns:
            options.extend(["display.width", columns])
    except OSError:
        pass

    if not truncate:
        options.extend(["display.max_colwidth", None])

    with pd.option_context(*options):
        if inside_notebook():
            from IPython.display import display

            display(df)
        else:
            print(df)

    if len(df) == limit:
        print(f"\n[Limited by {len(df)} rows]")

shuffle 

shuffle() -> Self

Shuffle the rows of the chain deterministically.

Source code in datachain/lib/dc/datachain.py 
2750
2751
2752
def shuffle(self) -> "Self":
    """Shuffle the rows of the chain deterministically."""
    return self.order_by("sys.rand")

subtract 

subtract(
    other: DataChain,
    on: str | Sequence[str] | None = None,
    right_on: str | Sequence[str] | None = None,
) -> Self

Remove rows that appear in another chain.

Parameters:

  • other (DataChain) –

    chain whose rows will be removed from self

  • on (str | Sequence[str] | None, default: None ) –

    columns to consider for determining row equality in self. If unspecified, defaults to all common columns between self and other.

  • right_on (str | Sequence[str] | None, default: None ) –

    columns to consider for determining row equality in other. If unspecified, defaults to the same values as on.

Source code in datachain/lib/dc/datachain.py 
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
def subtract(  # type: ignore[override]
    self,
    other: "DataChain",
    on: str | Sequence[str] | None = None,
    right_on: str | Sequence[str] | None = None,
) -> "Self":
    """Remove rows that appear in another chain.

    Parameters:
        other: chain whose rows will be removed from `self`
        on: columns to consider for determining row equality in `self`.
            If unspecified, defaults to all common columns
            between `self` and `other`.
        right_on: columns to consider for determining row equality in `other`.
            If unspecified, defaults to the same values as `on`.
    """
    if isinstance(on, str):
        if not on:
            raise DataChainParamsError("'on' cannot be an empty string")
        on = [on]
    elif isinstance(on, Sequence):
        if not on or any(not col for col in on):
            raise DataChainParamsError("'on' cannot contain empty strings")

    if isinstance(right_on, str):
        if not right_on:
            raise DataChainParamsError("'right_on' cannot be an empty string")
        right_on = [right_on]
    elif isinstance(right_on, Sequence):
        if not right_on or any(not col for col in right_on):
            raise DataChainParamsError("'right_on' cannot contain empty strings")

    if on is None and right_on is None:
        other_columns = set(other._effective_signals_schema.db_signals())
        signals = [
            c
            for c in self._effective_signals_schema.db_signals()
            if c in other_columns
        ]
        if not signals:
            raise DataChainParamsError("subtract(): no common columns")
    elif on is not None and right_on is None:
        right_on = on
        signals = list(self.signals_schema.resolve(*on).db_signals())
    elif on is None and right_on is not None:
        raise DataChainParamsError(
            "'on' must be specified when 'right_on' is provided"
        )
    else:
        if not isinstance(on, Sequence) or not isinstance(right_on, Sequence):
            raise TypeError(
                "'on' and 'right_on' must be 'str' or 'Sequence' object"
            )
        if len(on) != len(right_on):
            raise DataChainParamsError(
                "'on' and 'right_on' must have the same length"
            )
        signals = list(
            zip(
                self.signals_schema.resolve(*on).db_signals(),
                other.signals_schema.resolve(*right_on).db_signals(),
                strict=False,
            )  # type: ignore[arg-type]
        )
    return self._evolve(query=self._query.subtract(other._query, signals))  # type: ignore[arg-type]

sum 

sum(col: str) -> StandardType

Compute the sum of a column.

Parameters:

  • col (str) –

    The column to compute the sum for.

Returns:

  • StandardType –

    The sum of the column values.

Example 
total_size = chain.sum("file.size")
print(f"Total size: {total_size}")
Source code in datachain/lib/dc/datachain.py 
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
def sum(self, col: str) -> StandardType:  # type: ignore[override]
    """Compute the sum of a column.

    Parameters:
        col: The column to compute the sum for.

    Returns:
        The sum of the column values.

    Example:
        ```py
        total_size = chain.sum("file.size")
        print(f"Total size: {total_size}")
        ```
    """
    return self._extend_to_data_model("sum", col)

to_columnar_data_with_names 

to_columnar_data_with_names(
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
) -> tuple[list[str], Iterator[list[list[Any]]]]

Returns column names and the results as an iterator that provides chunks, with each chunk containing a list of columns, where each column contains a list of the row values for that column in that chunk. Useful for columnar data formats, such as parquet or other OLAP databases.

Source code in datachain/lib/dc/datachain.py 
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
def to_columnar_data_with_names(
    self, chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE
) -> tuple[list[str], Iterator[list[list[Any]]]]:
    """Returns column names and the results as an iterator that provides chunks,
    with each chunk containing a list of columns, where each column contains a
    list of the row values for that column in that chunk. Useful for columnar data
    formats, such as parquet or other OLAP databases.
    """
    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    def column_chunks() -> Iterator[list[list[Any]]]:
        for chunk_iter in batched_it(results_iter, chunk_size):
            columns: list[list[Any]] = [[] for _ in column_names]
            for row in chunk_iter:
                for i, col in enumerate(columns):
                    col.append(row[i])
            yield columns

    return column_names, column_chunks()

to_csv 

to_csv(
    path: str | PathLike[str],
    delimiter: str = ",",
    fs_kwargs: dict[str, Any] | None = None,
    **kwargs
) -> None

Save chain to a csv (comma-separated values) file.

Parameters:

  • path (str | PathLike[str]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • delimiter (str, default: ',' ) –

    Delimiter to use for the resulting file.

  • fs_kwargs (dict[str, Any] | None, default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
def to_csv(
    self,
    path: str | os.PathLike[str],
    delimiter: str = ",",
    fs_kwargs: dict[str, Any] | None = None,
    **kwargs,
) -> None:
    """Save chain to a csv (comma-separated values) file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        delimiter: Delimiter to use for the resulting file.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import csv

    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self._leaf_values()

    with opener(path, "w", newline="") as f:
        writer = csv.writer(f, delimiter=delimiter, **kwargs)
        writer.writerow(column_names)

        for row in results_iter:
            writer.writerow(row)

to_database 

to_database(
    table_name: str,
    connection: ConnectionType,
    *,
    batch_size: int = DEFAULT_DATABASE_BATCH_SIZE,
    on_conflict: str | None = None,
    conflict_columns: list[str] | None = None,
    column_mapping: dict[str, str | None] | None = None
) -> int

Save chain to a database table using a given database connection.

This method exports all DataChain records to a database table, creating the table if it doesn't exist and appending data if it does. The table schema is automatically inferred from the DataChain's signal schema.

For PostgreSQL, tables are created in the schema specified by the connection's search_path (defaults to 'public'). Use URL parameters to target specific schemas.

Parameters:

  • table_name (str) –

    Name of the database table to create/write to.

  • connection (ConnectionType) –

    SQLAlchemy connectable, str, or a sqlite3 connection Using SQLAlchemy makes it possible to use any DB supported by that library. If a DBAPI2 object, only sqlite3 is supported. The user is responsible for engine disposal and connection closure for the SQLAlchemy connectable; str connections are closed automatically.

  • batch_size (int, default: DEFAULT_DATABASE_BATCH_SIZE ) –

    Number of rows to insert per batch for optimal performance. Larger batches are faster but use more memory. Default: 10,000.

  • on_conflict (str | None, default: None ) –

    Strategy for handling duplicate rows (requires table constraints): - None: Raise error (sqlalchemy.exc.IntegrityError) on conflict (default) - "ignore": Skip duplicate rows silently - "update": Update existing rows with new values

  • conflict_columns (list[str] | None, default: None ) –

    List of column names that form a unique constraint for conflict resolution. Required when on_conflict='update' and using PostgreSQL.

  • column_mapping (dict[str, str | None] | None, default: None ) –

    Optional mapping to rename or skip columns: - Dict mapping DataChain column names to database column names - Set values to None to skip columns entirely, or use defaultdict to skip all columns except those specified.

Returns:

  • int ( int ) –

    Number of rows affected (inserted/updated). -1 if DB driver doesn't support telemetry.

Examples:

Basic usage with PostgreSQL: 

import datachain as dc

rows_affected = (dc
  .read_storage("s3://my-bucket/")
  .to_database("files_table", "postgresql://user:pass@localhost/mydb")
)
print(f"Inserted/updated {rows_affected} rows")

Using SQLite with connection string: 

rows_affected = chain.to_database("my_table", "sqlite:///data.db")
print(f"Affected {rows_affected} rows")

Column mapping and renaming: 

mapping = {
    "user.id": "id",
    "user.name": "name",
    "user.password": None  # Skip this column
}
chain.to_database("users", engine, column_mapping=mapping)

Handling conflicts (requires PRIMARY KEY or UNIQUE constraints): 

# Skip duplicates
chain.to_database("my_table", engine, on_conflict="ignore")

# Update existing records
chain.to_database(
   "my_table", engine, on_conflict="update", conflict_columns=["id"]
)

Working with different databases: 

# MySQL
mysql_engine = sa.create_engine("mysql+pymysql://user:pass@host/db")
chain.to_database("mysql_table", mysql_engine)

# SQLite in-memory
chain.to_database("temp_table", "sqlite:///:memory:")

PostgreSQL with schema support: 

pg_url = "postgresql://user:pass@host/db?options=-c search_path=analytics"
chain.to_database("processed_data", pg_url)

Source code in datachain/lib/dc/datachain.py 
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
def to_database(
    self,
    table_name: str,
    connection: "ConnectionType",
    *,
    batch_size: int = DEFAULT_DATABASE_BATCH_SIZE,
    on_conflict: str | None = None,
    conflict_columns: list[str] | None = None,
    column_mapping: dict[str, str | None] | None = None,
) -> int:
    """Save chain to a database table using a given database connection.

    This method exports all DataChain records to a database table, creating the
    table if it doesn't exist and appending data if it does. The table schema
    is automatically inferred from the DataChain's signal schema.

    For PostgreSQL, tables are created in the schema specified by the connection's
    search_path (defaults to 'public'). Use URL parameters to target specific
    schemas.

    Parameters:
        table_name: Name of the database table to create/write to.
        connection: SQLAlchemy connectable, str, or a sqlite3 connection
            Using SQLAlchemy makes it possible to use any DB supported by that
            library. If a DBAPI2 object, only sqlite3 is supported. The user is
            responsible for engine disposal and connection closure for the
            SQLAlchemy connectable; str connections are closed automatically.
        batch_size: Number of rows to insert per batch for optimal performance.
            Larger batches are faster but use more memory. Default: 10,000.
        on_conflict: Strategy for handling duplicate rows (requires table
            constraints):
            - None: Raise error (`sqlalchemy.exc.IntegrityError`) on conflict
              (default)
            - "ignore": Skip duplicate rows silently
            - "update": Update existing rows with new values
        conflict_columns: List of column names that form a unique constraint
            for conflict resolution. Required when on_conflict='update' and
            using PostgreSQL.
        column_mapping: Optional mapping to rename or skip columns:
            - Dict mapping DataChain column names to database column names
            - Set values to None to skip columns entirely, or use `defaultdict` to
              skip all columns except those specified.

    Returns:
        int: Number of rows affected (inserted/updated). -1 if DB driver doesn't
             support telemetry.

    Examples:
        Basic usage with PostgreSQL:
        ```py
        import datachain as dc

        rows_affected = (dc
          .read_storage("s3://my-bucket/")
          .to_database("files_table", "postgresql://user:pass@localhost/mydb")
        )
        print(f"Inserted/updated {rows_affected} rows")
        ```

        Using SQLite with connection string:
        ```py
        rows_affected = chain.to_database("my_table", "sqlite:///data.db")
        print(f"Affected {rows_affected} rows")
        ```

        Column mapping and renaming:
        ```py
        mapping = {
            "user.id": "id",
            "user.name": "name",
            "user.password": None  # Skip this column
        }
        chain.to_database("users", engine, column_mapping=mapping)
        ```

        Handling conflicts (requires PRIMARY KEY or UNIQUE constraints):
        ```py
        # Skip duplicates
        chain.to_database("my_table", engine, on_conflict="ignore")

        # Update existing records
        chain.to_database(
           "my_table", engine, on_conflict="update", conflict_columns=["id"]
        )
        ```

        Working with different databases:
        ```py
        # MySQL
        mysql_engine = sa.create_engine("mysql+pymysql://user:pass@host/db")
        chain.to_database("mysql_table", mysql_engine)

        # SQLite in-memory
        chain.to_database("temp_table", "sqlite:///:memory:")
        ```

        PostgreSQL with schema support:
        ```py
        pg_url = "postgresql://user:pass@host/db?options=-c search_path=analytics"
        chain.to_database("processed_data", pg_url)
        ```
    """
    from .database import to_database

    return to_database(
        self,
        table_name,
        connection,
        batch_size=batch_size,
        on_conflict=on_conflict,
        conflict_columns=conflict_columns,
        column_mapping=column_mapping,
    )

to_iter 

to_iter(*cols: str) -> Iterator[tuple[DataValue, ...]]

Yields rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Yields:

Example

Iterating over all rows: 

for row in ds.to_iter():
    print(row)

DataChain is iterable and can be used in a for loop directly which is equivalent to ds.to_iter(): 

for row in ds:
    print(row)

Iterating over all rows with selected columns: 

for name, size in ds.to_iter("file.path", "file.size"):
    print(name, size)

Iterating over a single column: 

for (file,) in ds.to_iter("file.path"):
    print(file)

Source code in datachain/lib/dc/datachain.py 
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
def to_iter(self, *cols: str) -> Iterator[tuple[DataValue, ...]]:
    """Yields rows of values, optionally limited to the specified columns.

    Args:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Yields:
        (tuple[DataType, ...]): Yields a tuple of items for each row.

    Example:
        Iterating over all rows:
        ```py
        for row in ds.to_iter():
            print(row)
        ```

        DataChain is iterable and can be used in a for loop directly which is
        equivalent to `ds.to_iter()`:
        ```py
        for row in ds:
            print(row)
        ```

        Iterating over all rows with selected columns:
        ```py
        for name, size in ds.to_iter("file.path", "file.size"):
            print(name, size)
        ```

        Iterating over a single column:
        ```py
        for (file,) in ds.to_iter("file.path"):
            print(file)
        ```
    """
    chain = self.select(*cols) if cols else self
    signals_schema = chain._effective_signals_schema
    db_signals = signals_schema.db_signals()
    with self._query.ordered_select(*db_signals).as_iterable() as rows:
        for row in rows:
            ret = signals_schema.row_to_features(
                row, catalog=chain.session.catalog, cache=chain._settings.cache
            )
            yield tuple(ret)

to_json 

to_json(
    path: str | PathLike[str],
    fs_kwargs: dict[str, Any] | None = None,
    include_outer_list: bool = True,
) -> None

Save chain to a JSON file.

Parameters:

  • path (str | PathLike[str]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs (dict[str, Any] | None, default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

  • include_outer_list (bool, default: True ) –

    Sets whether to include an outer list for all rows. Setting this to True makes the file valid JSON, while False instead writes in the JSON lines format.

Source code in datachain/lib/dc/datachain.py 
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
def to_json(
    self,
    path: str | os.PathLike[str],
    fs_kwargs: dict[str, Any] | None = None,
    include_outer_list: bool = True,
) -> None:
    """Save chain to a JSON file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
        include_outer_list: Sets whether to include an outer list for all rows.
            Setting this to True makes the file valid JSON, while False instead
            writes in the JSON lines format.
    """
    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    headers = [list(filter(None, header)) for header in headers]

    is_first = True

    with opener(path, "wb") as f:
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"[\n")
        for row in self._leaf_values():
            if not is_first:
                if include_outer_list:
                    # This makes the file JSON instead of JSON lines.
                    f.write(b",\n")
                else:
                    f.write(b"\n")
            else:
                is_first = False
            f.write(
                json.dumps(
                    row_to_nested_dict(headers, row), ensure_ascii=False
                ).encode("utf-8")
            )
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"\n]\n")

to_jsonl 

to_jsonl(
    path: str | PathLike[str],
    fs_kwargs: dict[str, Any] | None = None,
) -> None

Save chain to a JSON lines file.

Parameters:

  • path (str | PathLike[str]) –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs (dict[str, Any] | None, default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
def to_jsonl(
    self,
    path: str | os.PathLike[str],
    fs_kwargs: dict[str, Any] | None = None,
) -> None:
    """Save chain to a JSON lines file.

    Parameters:
        path: Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    self.to_json(path, fs_kwargs, include_outer_list=False)

to_list 

to_list(*cols: str) -> list[tuple[DataValue, ...]]

Returns a list of rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Returns:

  • list[tuple[DataValue, ...]] –

    list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

Example

Getting all rows as a list: 

rows = dc.to_list()
print(rows)

Getting all rows with selected columns as a list: 

name_size_pairs = dc.to_list("file.path", "file.size")
print(name_size_pairs)

Getting a single column as a list: 

files = dc.to_list("file.path")
print(files)  # Returns list of 1-tuples

Source code in datachain/lib/dc/datachain.py 
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
def to_list(self, *cols: str) -> list[tuple[DataValue, ...]]:
    """Returns a list of rows of values, optionally limited to the specified
    columns.

    Parameters:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Returns:
        list[tuple[DataType, ...]]: Returns a list of tuples of items for each row.

    Example:
        Getting all rows as a list:
        ```py
        rows = dc.to_list()
        print(rows)
        ```

        Getting all rows with selected columns as a list:
        ```py
        name_size_pairs = dc.to_list("file.path", "file.size")
        print(name_size_pairs)
        ```

        Getting a single column as a list:
        ```py
        files = dc.to_list("file.path")
        print(files)  # Returns list of 1-tuples
        ```
    """
    return list(self.to_iter(*cols))

to_pandas 

to_pandas(
    flatten: bool = False,
    include_hidden: bool = True,
    as_object: bool = False,
) -> DataFrame

Return a pandas DataFrame from the chain.

Parameters:

  • flatten (bool, default: False ) –

    Whether to use a multiindex or flatten column names.

  • include_hidden (bool, default: True ) –

    Whether to include hidden columns.

  • as_object (bool, default: False ) –

    Whether to emit a dataframe backed by Python objects rather than pandas-inferred dtypes.

Returns:

  • DataFrame –

    pd.DataFrame: A pandas DataFrame representation of the chain.

Source code in datachain/lib/dc/datachain.py 
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
def to_pandas(
    self,
    flatten: bool = False,
    include_hidden: bool = True,
    as_object: bool = False,
) -> "pd.DataFrame":
    """Return a pandas DataFrame from the chain.

    Parameters:
        flatten: Whether to use a multiindex or flatten column names.
        include_hidden: Whether to include hidden columns.
        as_object: Whether to emit a dataframe backed by Python objects
            rather than pandas-inferred dtypes.

    Returns:
        pd.DataFrame: A pandas DataFrame representation of the chain.
    """
    import pandas as pd

    headers, max_length = self._effective_signals_schema.get_headers_with_length(
        include_hidden=include_hidden
    )

    columns: list[str] | pd.MultiIndex
    if flatten or max_length < 2:
        columns = [".".join(filter(None, header)) for header in headers]
    else:
        columns = pd.MultiIndex.from_tuples(map(tuple, headers))

    results = self.results(include_hidden=include_hidden)
    if as_object:
        df = pd.DataFrame(results, columns=columns, dtype=object)
        df.where(pd.notna(df), None, inplace=True)
        return df
    return pd.DataFrame.from_records(results, columns=columns)

to_parquet 

to_parquet(
    path: str | PathLike[str] | BinaryIO,
    partition_cols: Sequence[str] | None = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: dict[str, Any] | None = None,
    **kwargs
) -> None

Save chain to parquet file with SignalSchema metadata.

Parameters:

  • path (str | PathLike[str] | BinaryIO) –

    Path or a file-like binary object to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • partition_cols (Sequence[str] | None, default: None ) –

    Column names by which to partition the dataset.

  • chunk_size (int, default: DEFAULT_PARQUET_CHUNK_SIZE ) –

    The chunk size of results to read and convert to columnar data, to avoid running out of memory.

  • fs_kwargs (dict[str, Any] | None, default: None ) –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
def to_parquet(
    self,
    path: str | os.PathLike[str] | BinaryIO,
    partition_cols: Sequence[str] | None = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: dict[str, Any] | None = None,
    **kwargs,
) -> None:
    """Save chain to parquet file with SignalSchema metadata.

    Parameters:
        path: Path or a file-like binary object to save the file. This supports
            local paths as well as remote paths, such as s3:// or hf:// with fsspec.
        partition_cols: Column names by which to partition the dataset.
        chunk_size: The chunk size of results to read and convert to columnar
            data, to avoid running out of memory.
        fs_kwargs: Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import pyarrow as pa
    import pyarrow.parquet as pq

    from datachain.lib.arrow import DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY

    fsspec_fs = None

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        if path.startswith("file://"):
            # pyarrow does not handle file:// uris, and needs a direct path instead.
            from urllib.parse import urlparse

            path = urlparse(path).path
            if sys.platform == "win32":
                path = os.path.normpath(path.lstrip("/"))

        fsspec_fs = client.create_fs(**fs_kwargs)

    _partition_cols = list(partition_cols) if partition_cols else None
    signal_schema_metadata = json.dumps(
        self._effective_signals_schema.serialize(), ensure_ascii=False
    ).encode("utf-8")

    column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)

    parquet_schema = None
    parquet_writer = None
    first_chunk = True

    for chunk in column_chunks:
        # pyarrow infers the best parquet schema from the python types of
        # the input data.
        table = pa.Table.from_pydict(
            dict(zip(column_names, chunk, strict=False)),
            schema=parquet_schema,
        )

        # Preserve any existing metadata, and add the DataChain SignalSchema.
        existing_metadata = table.schema.metadata or {}
        merged_metadata = {
            **existing_metadata,
            DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY: signal_schema_metadata,
        }
        table = table.replace_schema_metadata(merged_metadata)
        parquet_schema = table.schema

        if _partition_cols:
            # Write to a partitioned parquet dataset.
            pq.write_to_dataset(
                table,
                root_path=path,
                partition_cols=_partition_cols,
                filesystem=fsspec_fs,
                **kwargs,
            )
        else:
            if first_chunk:
                # Write to a single parquet file.
                parquet_writer = pq.ParquetWriter(
                    path, parquet_schema, filesystem=fsspec_fs, **kwargs
                )
                first_chunk = False

            assert parquet_writer
            parquet_writer.write_table(table)

    if parquet_writer:
        parquet_writer.close()

to_pytorch 

to_pytorch(
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
)

Convert to pytorch dataset format.

Parameters:

  • transform (Transform, default: None ) –

    Torchvision transforms to apply to the dataset.

  • tokenizer (Callable, default: None ) –

    Tokenizer to use to tokenize text values.

  • tokenizer_kwargs (dict, default: None ) –

    Additional kwargs to pass when calling tokenizer.

  • num_samples (int, default: 0 ) –

    Number of random samples to draw for each epoch. This argument is ignored if num_samples=0 (the default).

  • remove_prefetched (bool, default: False ) –

    Whether to remove prefetched files after reading.

Example 
from torch.utils.data import DataLoader
loader = DataLoader(
    chain.select("file", "label").to_pytorch(),
    batch_size=16
)
Source code in datachain/lib/dc/datachain.py 
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
def to_pytorch(
    self,
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
):
    """Convert to pytorch dataset format.

    Args:
        transform (Transform): Torchvision transforms to apply to the dataset.
        tokenizer (Callable): Tokenizer to use to tokenize text values.
        tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
        num_samples (int): Number of random samples to draw for each epoch.
            This argument is ignored if `num_samples=0` (the default).
        remove_prefetched (bool): Whether to remove prefetched files after reading.

    Example:
        ```py
        from torch.utils.data import DataLoader
        loader = DataLoader(
            chain.select("file", "label").to_pytorch(),
            batch_size=16
        )
        ```
    """
    from datachain.torch import PytorchDataset

    if self._query.attached:
        chain = self
    else:
        chain = self.persist()
    assert chain.name is not None  # for mypy
    return PytorchDataset(
        chain.name,
        chain.version,
        catalog=self.session.catalog,
        transform=transform,
        tokenizer=tokenizer,
        tokenizer_kwargs=tokenizer_kwargs,
        num_samples=num_samples,
        dc_settings=chain._settings,
        remove_prefetched=remove_prefetched,
    )

to_records 

to_records() -> list[dict[str, Any]]

Convert every row to a dictionary.

Source code in datachain/lib/dc/datachain.py 
1483
1484
1485
1486
1487
1488
1489
def to_records(self) -> list[dict[str, Any]]:
    """Convert every row to a dictionary."""

    def to_dict(cols: list[str], row: tuple[Any, ...]) -> dict[str, Any]:
        return dict(zip(cols, row, strict=False))

    return self.results(row_factory=to_dict)

to_storage 

to_storage(
    output: str | PathLike[str],
    signal: str = "file",
    placement: ExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: int | None = EXPORT_FILES_MAX_THREADS,
    anon: bool | None = None,
    client_config: dict | None = None,
) -> None

Export files from a specified signal to a directory. Files can be exported to a local or cloud directory.

Parameters:

  • output (str | PathLike[str]) –

    Path to the target directory for exporting files.

  • signal (str, default: 'file' ) –

    Name of the signal to export files from.

  • placement (ExportPlacement, default: 'fullpath' ) –

    The method to use for naming exported files. The possible values are: "filename", "etag", "fullpath", and "checksum".

  • link_type (Literal['copy', 'symlink'], default: 'copy' ) –

    Method to use for exporting files. Falls back to 'copy' if symlinking fails.

  • num_threads (int | None, default: EXPORT_FILES_MAX_THREADS ) –

    number of threads to use for exporting files. By default, it uses 5 threads.

  • anon (bool | None, default: None ) –

    If True, we will treat cloud bucket as a public one. Default behavior depends on the previous session configuration (e.g. happens in the initial read_storage) and particular cloud storage client implementation (e.g. S3 fallbacks to anonymous access if no credentials were found).

  • client_config (dict | None, default: None ) –

    Optional configuration for the destination storage client

Example

Cross cloud transfer 

import datachain as dc

ds = dc.read_storage("s3://mybucket")
ds.to_storage("gs://mybucket", placement="filename")

Source code in datachain/lib/dc/datachain.py 
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
def to_storage(
    self,
    output: str | os.PathLike[str],
    signal: str = "file",
    placement: FileExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: int | None = EXPORT_FILES_MAX_THREADS,
    anon: bool | None = None,
    client_config: dict | None = None,
) -> None:
    """Export files from a specified signal to a directory. Files can be
    exported to a local or cloud directory.

    Args:
        output: Path to the target directory for exporting files.
        signal: Name of the signal to export files from.
        placement: The method to use for naming exported files.
            The possible values are: "filename", "etag", "fullpath", and "checksum".
        link_type: Method to use for exporting files.
            Falls back to `'copy'` if symlinking fails.
        num_threads: number of threads to use for exporting files.
            By default, it uses 5 threads.
        anon: If True, we will treat cloud bucket as a public one. Default behavior
            depends on the previous session configuration (e.g. happens in the
            initial `read_storage`) and particular cloud storage client
            implementation (e.g. S3 fallbacks to anonymous access if no credentials
            were found).
        client_config: Optional configuration for the destination storage client

    Example:
        Cross cloud transfer
        ```py
        import datachain as dc

        ds = dc.read_storage("s3://mybucket")
        ds.to_storage("gs://mybucket", placement="filename")
        ```
    """
    chain = self.persist()
    count = chain.count()

    if placement == "filename" and (
        chain._query.distinct(pathfunc.name(C(f"{signal}__path"))).count() != count
    ):
        raise ValueError("Files with the same name found")

    if anon is not None:
        client_config = (client_config or {}) | {"anon": anon}

    progress_bar = tqdm(
        desc=f"Exporting files to {output}: ",
        unit=" files",
        unit_scale=True,
        unit_divisor=10,
        total=count,
        leave=False,
    )
    file_exporter = FileExporter(
        output,
        placement,
        self._settings.cache if self._settings else False,
        link_type,
        max_threads=num_threads or 1,
        client_config=client_config,
    )
    file_exporter.run(
        (rows[0] for rows in chain.to_iter(signal)),
        progress_bar,
    )

to_values 

to_values(col: str) -> list[DataValue]

Returns a flat list of values from a single column.

Parameters:

  • col (str) –

    The name of the column to extract values from.

Returns:

  • list[DataValue] –

    list[DataValue]: Returns a flat list of values from the specified column.

Example

Getting all values from a single column: 

file_paths = dc.to_values("file.path")
print(file_paths)  # Returns list of strings

Getting all file sizes: 

sizes = dc.to_values("file.size")
print(sizes)  # Returns list of integers

Source code in datachain/lib/dc/datachain.py 
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
def to_values(self, col: str) -> list[DataValue]:
    """Returns a flat list of values from a single column.

    Parameters:
        col: The name of the column to extract values from.

    Returns:
        list[DataValue]: Returns a flat list of values from the specified column.

    Example:
        Getting all values from a single column:
        ```py
        file_paths = dc.to_values("file.path")
        print(file_paths)  # Returns list of strings
        ```

        Getting all file sizes:
        ```py
        sizes = dc.to_values("file.size")
        print(sizes)  # Returns list of integers
        ```
    """
    return [row[0] for row in self.to_list(col)]

union 

union(other: Self) -> Self

Return the set union of the two datasets.

Parameters:

  • other (Self) –

    chain whose rows will be added to self.

Source code in datachain/lib/dc/datachain.py 
1719
1720
1721
1722
1723
1724
1725
1726
@delta_disabled
def union(self, other: "Self") -> "Self":
    """Return the set union of the two datasets.

    Parameters:
        other: chain whose rows will be added to `self`.
    """
    return self._evolve(query=self._query.union(other._query))

DataChainError

Bases: Exception

Session 

Session(
    name="",
    catalog: Catalog | None = None,
    client_config: dict | None = None,
    in_memory: bool = False,
)

Session is a context that keeps track of temporary DataChain datasets for a proper cleanup. By default, a global session is created.

Temporary or ephemeral datasets are the ones created without specified name. They are useful for optimization purposes and should be automatically removed.

Temp dataset has specific name format

"session_"

The suffix is optional. Both s are auto-generated.

Temp dataset examples

session_myname_624b41_48e8b4 session_4b962d_2a5dff

Parameters:

name (str): The name of the session. Only latters and numbers are supported. It can be empty. catalog (Catalog): Catalog object.

Source code in datachain/query/session.py 
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def __init__(
    self,
    name="",
    catalog: "Catalog | None" = None,
    client_config: dict | None = None,
    in_memory: bool = False,
):
    if re.match(r"^[0-9a-zA-Z]*$", name) is None:
        raise ValueError(
            f"Session name can contain only letters or numbers - '{name}' given."
        )

    if not name:
        name = self.GLOBAL_SESSION_NAME

    session_uuid = uuid4().hex[: self.SESSION_UUID_LEN]
    self.name = f"{name}_{session_uuid}"
    self.is_new_catalog = not catalog
    self.catalog = catalog or get_catalog(
        client_config=client_config, in_memory=in_memory
    )
    self.dataset_versions: list[tuple[DatasetRecord, str, bool]] = []

get classmethod 

get(
    session: Session | None = None,
    catalog: Catalog | None = None,
    client_config: dict | None = None,
    in_memory: bool = False,
) -> Session

Creates a Session() object from a catalog.

Parameters:

  • session (Session, default: None ) –

    Optional Session(). If not provided a new session will be created. It's needed mostly for simple API purposes.

  • catalog (Catalog, default: None ) –

    Optional catalog. By default, a new catalog is created.

Source code in datachain/query/session.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
@classmethod
def get(
    cls,
    session: "Session | None" = None,
    catalog: "Catalog | None" = None,
    client_config: dict | None = None,
    in_memory: bool = False,
) -> "Session":
    """Creates a Session() object from a catalog.

    Parameters:
        session (Session): Optional Session(). If not provided a new session will
                be created. It's needed mostly for simple API purposes.
        catalog (Catalog): Optional catalog. By default, a new catalog is created.
    """
    if session:
        return session

    # Access the active (most recent) context from the stack
    if cls.SESSION_CONTEXTS:
        session = cls.SESSION_CONTEXTS[-1]

    elif cls.GLOBAL_SESSION_CTX is None:
        cls.GLOBAL_SESSION_CTX = Session(
            cls.GLOBAL_SESSION_NAME,
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session = cls.GLOBAL_SESSION_CTX

        atexit.register(cls._global_cleanup)
        cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
        sys.excepthook = cls.except_hook
    else:
        session = cls.GLOBAL_SESSION_CTX

    if client_config and session.catalog.client_config != client_config:
        session = Session(
            "session" + uuid4().hex[:4],
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session.__enter__()

    return session

Sys

Bases: DataModel

Model for internal DataChain signals id and rand.