Skip to content

DataChain

The DataChain class creates a data chain, which is a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data. See DataChain for examples of how to create a chain.

C module-attribute 

C = Column

Column 

Column(
    text, type_=None, is_literal=False, _selectable=None
)

Bases: ColumnClause

Source code in datachain/query/schema.py 
43
44
45
46
47
48
def __init__(self, text, type_=None, is_literal=False, _selectable=None):
    """Dataset column."""
    self.name = ColumnMeta.to_db_name(text)
    super().__init__(
        self.name, type_=type_, is_literal=is_literal, _selectable=_selectable
    )

glob 

glob(glob_str)

Search for matches using glob pattern matching.

Source code in datachain/query/schema.py 
53
54
55
def glob(self, glob_str):
    """Search for matches using glob pattern matching."""
    return self.op("GLOB")(glob_str)

regexp 

regexp(regexp_str)

Search for matches using regexp pattern matching.

Source code in datachain/query/schema.py 
57
58
59
def regexp(self, regexp_str):
    """Search for matches using regexp pattern matching."""
    return self.op("REGEXP")(regexp_str)

from_csv 

from_csv(
    path,
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    nrows=None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[
        dict[str, Union[str, DataType]]
    ] = None,
    parse_options: Optional[
        dict[str, Union[str, Union[bool, Callable]]]
    ] = None,
    **kwargs
) -> DataChain

Generate chain from csv files.

Parameters:

  • path –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • delimiter –

    Character for delimiting columns. Takes precedence if also specified in parse_options. Defaults to ",".

  • header –

    Whether the files include a header row.

  • output –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • object_name –

    Created object column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • nrows –

    Optional row limit.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • column_types –

    Dictionary of column names and their corresponding types. It is passed to CSV reader and for each column specified type auto inference is disabled.

  • parse_options (Optional[dict[str, Union[str, Union[bool, Callable]]]], default: None ) –

    Tells the parser how to process lines. See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

Example

Reading a csv file: 

import datachain as dc
chain = dc.from_csv("s3://mybucket/file.csv")

Reading csv files from a directory as a combined dataset: 

import datachain as dc
chain = dc.from_csv("s3://mybucket/dir")

Source code in datachain/lib/dc/csv.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def from_csv(
    path,
    delimiter: Optional[str] = None,
    header: bool = True,
    output: OutputType = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    nrows=None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    column_types: Optional[dict[str, "Union[str, ArrowDataType]"]] = None,
    parse_options: Optional[dict[str, "Union[str, Union[bool, Callable]]"]] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from csv files.

    Parameters:
        path : Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        delimiter : Character for delimiting columns. Takes precedence if also
            specified in `parse_options`. Defaults to ",".
        header : Whether the files include a header row.
        output : Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        object_name : Created object column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        nrows : Optional row limit.
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        column_types : Dictionary of column names and their corresponding types.
            It is passed to CSV reader and for each column specified type auto
            inference is disabled.
        parse_options: Tells the parser how to process lines.
            See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html

    Example:
        Reading a csv file:
        ```py
        import datachain as dc
        chain = dc.from_csv("s3://mybucket/file.csv")
        ```

        Reading csv files from a directory as a combined dataset:
        ```py
        import datachain as dc
        chain = dc.from_csv("s3://mybucket/dir")
        ```
    """
    from pandas.io.parsers.readers import STR_NA_VALUES
    from pyarrow.csv import ConvertOptions, ParseOptions, ReadOptions
    from pyarrow.dataset import CsvFileFormat
    from pyarrow.lib import type_for_alias

    from .datachain import DataChain

    parse_options = parse_options or {}
    if "delimiter" not in parse_options:
        parse_options["delimiter"] = ","
    if delimiter:
        parse_options["delimiter"] = delimiter

    if column_types:
        column_types = {
            name: type_for_alias(typ) if isinstance(typ, str) else typ
            for name, typ in column_types.items()
        }
    else:
        column_types = {}

    chain = DataChain.from_storage(path, session=session, settings=settings, **kwargs)

    column_names = None
    if not header:
        if not output:
            msg = "error parsing csv - provide output if no header"
            raise DatasetPrepareError(chain.name, msg)
        if isinstance(output, Sequence):
            column_names = output  # type: ignore[assignment]
        elif isinstance(output, dict):
            column_names = list(output.keys())
        elif (fr := ModelStore.to_pydantic(output)) is not None:
            column_names = list(fr.model_fields.keys())
        else:
            msg = f"error parsing csv - incompatible output type {type(output)}"
            raise DatasetPrepareError(chain.name, msg)

    parse_options = ParseOptions(**parse_options)
    read_options = ReadOptions(column_names=column_names)
    convert_options = ConvertOptions(
        strings_can_be_null=True,
        null_values=STR_NA_VALUES,
        column_types=column_types,
    )
    format = CsvFileFormat(
        parse_options=parse_options,
        read_options=read_options,
        convert_options=convert_options,
    )
    return chain.parse_tabular(
        output=output,
        object_name=object_name,
        model_name=model_name,
        source=source,
        nrows=nrows,
        format=format,
    )

from_dataset 

from_dataset(
    name: str,
    version: Optional[int] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    fallback_to_studio: bool = True,
) -> DataChain

Get data from a saved Dataset. It returns the chain itself. If dataset or version is not found locally, it will try to pull it from Studio.

Parameters:

  • name –

    dataset name

  • version –

    dataset version

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • fallback_to_studio –

    Try to pull dataset from Studio if not found locally. Default is True.

Example 
import datachain as dc
chain = dc.from_dataset("my_cats")

chain = dc.from_dataset("my_cats", fallback_to_studio=False)

chain = dc.from_dataset("my_cats", version=1)

session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
settings = {
    "cache": True,
    "parallel": 4,
    "workers": 4,
    "min_task_size": 1000,
    "prefetch": 10,
}
chain = dc.from_dataset(
    name="my_cats",
    version=1,
    session=session,
    settings=settings,
    fallback_to_studio=True,
)
Source code in datachain/lib/dc/datasets.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def from_dataset(
    name: str,
    version: Optional[int] = None,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    fallback_to_studio: bool = True,
) -> "DataChain":
    """Get data from a saved Dataset. It returns the chain itself.
    If dataset or version is not found locally, it will try to pull it from Studio.

    Parameters:
        name : dataset name
        version : dataset version
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        fallback_to_studio : Try to pull dataset from Studio if not found locally.
            Default is True.

    Example:
        ```py
        import datachain as dc
        chain = dc.from_dataset("my_cats")
        ```

        ```py
        chain = dc.from_dataset("my_cats", fallback_to_studio=False)
        ```

        ```py
        chain = dc.from_dataset("my_cats", version=1)
        ```

        ```py
        session = Session.get(client_config={"aws_endpoint_url": "<minio-url>"})
        settings = {
            "cache": True,
            "parallel": 4,
            "workers": 4,
            "min_task_size": 1000,
            "prefetch": 10,
        }
        chain = dc.from_dataset(
            name="my_cats",
            version=1,
            session=session,
            settings=settings,
            fallback_to_studio=True,
        )
        ```
    """
    from datachain.telemetry import telemetry

    from .datachain import DataChain

    query = DatasetQuery(
        name=name,
        version=version,
        session=session,
        indexing_column_types=File._datachain_column_types,
        fallback_to_studio=fallback_to_studio,
    )
    telemetry.send_event_once("class", "datachain_init", name=name, version=version)
    if settings:
        _settings = Settings(**settings)
    else:
        _settings = Settings()

    signals_schema = SignalSchema({"sys": Sys})
    if query.feature_schema:
        signals_schema |= SignalSchema.deserialize(query.feature_schema)
    else:
        signals_schema |= SignalSchema.from_column_types(query.column_types or {})
    return DataChain(query, _settings, signals_schema)

datasets 

datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    object_name: str = "dataset",
    include_listing: bool = False,
    studio: bool = False,
) -> DataChain

Generate chain with list of registered datasets.

Parameters:

  • session (Optional[Session], default: None ) –

    Optional session instance. If not provided, uses default session.

  • settings (Optional[dict], default: None ) –

    Optional dictionary of settings to configure the chain.

  • in_memory (bool, default: False ) –

    If True, creates an in-memory session. Defaults to False.

  • object_name (str, default: 'dataset' ) –

    Name of the output object in the chain. Defaults to "dataset".

  • include_listing (bool, default: False ) –

    If True, includes listing datasets. Defaults to False.

  • studio (bool, default: False ) –

    If True, returns datasets from Studio only, otherwise returns all local datasets. Defaults to False.

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance containing dataset information.

Example 
import datachain as dc

chain = dc.datasets()
for ds in chain.collect("dataset"):
    print(f"{ds.name}@v{ds.version}")
Source code in datachain/lib/dc/datasets.py 
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def datasets(
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    object_name: str = "dataset",
    include_listing: bool = False,
    studio: bool = False,
) -> "DataChain":
    """Generate chain with list of registered datasets.

    Args:
        session: Optional session instance. If not provided, uses default session.
        settings: Optional dictionary of settings to configure the chain.
        in_memory: If True, creates an in-memory session. Defaults to False.
        object_name: Name of the output object in the chain. Defaults to "dataset".
        include_listing: If True, includes listing datasets. Defaults to False.
        studio: If True, returns datasets from Studio only,
            otherwise returns all local datasets. Defaults to False.

    Returns:
        DataChain: A new DataChain instance containing dataset information.

    Example:
        ```py
        import datachain as dc

        chain = dc.datasets()
        for ds in chain.collect("dataset"):
            print(f"{ds.name}@v{ds.version}")
        ```
    """

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    datasets_values = [
        DatasetInfo.from_models(d, v, j)
        for d, v, j in catalog.list_datasets_versions(
            include_listing=include_listing, studio=studio
        )
    ]

    return from_values(
        session=session,
        settings=settings,
        in_memory=in_memory,
        output={object_name: DatasetInfo},
        **{object_name: datasets_values},  # type: ignore[arg-type]
    )

from_hf 

from_hf(
    dataset: Union[str, HFDatasetType],
    *args,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    object_name: str = "",
    model_name: str = "",
    **kwargs
) -> DataChain

Generate chain from huggingface hub dataset.

Parameters:

  • dataset –

    Path or name of the dataset to read from Hugging Face Hub, or an instance of datasets.Dataset-like object.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

  • object_name –

    Generated object column name.

  • model_name –

    Generated model name.

  • kwargs –

    Parameters to pass to datasets.load_dataset.

Example

Load from Hugging Face Hub: 

import datachain as dc
chain = dc.from_hf("beans", split="train")

Generate chain from loaded dataset: 

from datasets import load_dataset
ds = load_dataset("beans", split="train")
import datachain as dc
chain = dc.from_hf(ds)

Source code in datachain/lib/dc/hf.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def from_hf(
    dataset: Union[str, "HFDatasetType"],
    *args,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    object_name: str = "",
    model_name: str = "",
    **kwargs,
) -> "DataChain":
    """Generate chain from huggingface hub dataset.

    Parameters:
        dataset : Path or name of the dataset to read from Hugging Face Hub,
            or an instance of `datasets.Dataset`-like object.
        session : Session to use for the chain.
        settings : Settings to use for the chain.
        object_name : Generated object column name.
        model_name : Generated model name.
        kwargs : Parameters to pass to datasets.load_dataset.

    Example:
        Load from Hugging Face Hub:
        ```py
        import datachain as dc
        chain = dc.from_hf("beans", split="train")
        ```

        Generate chain from loaded dataset:
        ```py
        from datasets import load_dataset
        ds = load_dataset("beans", split="train")
        import datachain as dc
        chain = dc.from_hf(ds)
        ```
    """
    from datachain.lib.hf import HFGenerator, get_output_schema, stream_splits

    from .datachain import DataChain

    output: dict[str, DataType] = {}
    ds_dict = stream_splits(dataset, *args, **kwargs)
    if len(ds_dict) > 1:
        output = {"split": str}

    model_name = model_name or object_name or ""
    hf_features = next(iter(ds_dict.values())).features
    output = output | get_output_schema(hf_features)
    model = dict_to_data_model(model_name, output)
    if object_name:
        output = {object_name: model}

    chain = DataChain.from_values(
        split=list(ds_dict.keys()), session=session, settings=settings
    )
    return chain.gen(HFGenerator(dataset, model, *args, **kwargs), output=output)

from_json 

from_json(
    path: Union[str, PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    object_name: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows=None,
    **kwargs
) -> DataChain

Get data from JSON. It returns the chain itself.

Parameters:

  • path –

    storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type –

    read file as "binary", "text", or "image" data. Default is "text".

  • spec –

    optional Data Model

  • schema_from –

    path to sample to infer spec (if schema not provided)

  • object_name –

    generated object column name

  • model_name –

    optional generated model name

  • format (Optional[str], default: 'json' ) –

    "json", "jsonl"

  • jmespath –

    optional JMESPATH expression to reduce JSON

  • nrows –

    optional row limit for jsonl and JSON arrays

Example

infer JSON schema from data, reduce using JMESPATH 

import datachain as dc
chain = dc.from_json("gs://json", jmespath="key1.key2")

infer JSON schema from a particular path 

import datachain as dc
chain = dc.from_json("gs://json_ds", schema_from="gs://json/my.json")

Source code in datachain/lib/dc/json.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def from_json(
    path: Union[str, os.PathLike[str]],
    type: FileType = "text",
    spec: Optional[DataType] = None,
    schema_from: Optional[str] = "auto",
    jmespath: Optional[str] = None,
    object_name: Optional[str] = "",
    model_name: Optional[str] = None,
    format: Optional[str] = "json",
    nrows=None,
    **kwargs,
) -> "DataChain":
    """Get data from JSON. It returns the chain itself.

    Parameters:
        path : storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type : read file as "binary", "text", or "image" data. Default is "text".
        spec : optional Data Model
        schema_from : path to sample to infer spec (if schema not provided)
        object_name : generated object column name
        model_name : optional generated model name
        format: "json", "jsonl"
        jmespath : optional JMESPATH expression to reduce JSON
        nrows : optional row limit for jsonl and JSON arrays

    Example:
        infer JSON schema from data, reduce using JMESPATH
        ```py
        import datachain as dc
        chain = dc.from_json("gs://json", jmespath="key1.key2")
        ```

        infer JSON schema from a particular path
        ```py
        import datachain as dc
        chain = dc.from_json("gs://json_ds", schema_from="gs://json/my.json")
        ```
    """
    from .datachain import DataChain

    if schema_from == "auto":
        schema_from = str(path)

    def jmespath_to_name(s: str):
        name_end = re.search(r"\W", s).start() if re.search(r"\W", s) else len(s)  # type: ignore[union-attr]
        return s[:name_end]

    if (not object_name) and jmespath:
        object_name = jmespath_to_name(jmespath)
    if not object_name:
        object_name = format
    chain = DataChain.from_storage(uri=path, type=type, **kwargs)
    signal_dict = {
        object_name: read_meta(
            schema_from=schema_from,
            format=format,
            spec=spec,
            model_name=model_name,
            jmespath=jmespath,
            nrows=nrows,
        ),
        "params": {"file": File},
    }
    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}
    return chain.settings(**settings).gen(**signal_dict)  # type: ignore[misc, arg-type]

listings 

listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    object_name: str = "listing",
    **kwargs
) -> DataChain

Generate chain with list of cached listings. Listing is a special kind of dataset which has directory listing data of some underlying storage (e.g S3 bucket).

Example 
import datachain as dc
dc.listings().show()
Source code in datachain/lib/dc/listings.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
def listings(
    session: Optional[Session] = None,
    in_memory: bool = False,
    object_name: str = "listing",
    **kwargs,
) -> "DataChain":
    """Generate chain with list of cached listings.
    Listing is a special kind of dataset which has directory listing data of
    some underlying storage (e.g S3 bucket).

    Example:
        ```py
        import datachain as dc
        dc.listings().show()
        ```
    """
    session = Session.get(session, in_memory=in_memory)
    catalog = kwargs.get("catalog") or session.catalog

    return from_values(
        session=session,
        in_memory=in_memory,
        output={object_name: ListingInfo},
        **{object_name: catalog.listings()},  # type: ignore[arg-type]
    )

from_pandas 

from_pandas(
    df: DataFrame,
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    object_name: str = "",
) -> DataChain

Generate chain from pandas data-frame.

Example 
import pandas as pd
import datachain as dc

df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
dc.from_pandas(df)
Source code in datachain/lib/dc/pandas.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def from_pandas(  # type: ignore[override]
    df: "pd.DataFrame",
    name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    object_name: str = "",
) -> "DataChain":
    """Generate chain from pandas data-frame.

    Example:
        ```py
        import pandas as pd
        import datachain as dc

        df = pd.DataFrame({"fib": [1, 2, 3, 5, 8]})
        dc.from_pandas(df)
        ```
    """
    from .utils import DatasetPrepareError

    fr_map = {col.lower(): df[col].tolist() for col in df.columns}

    for column in fr_map:
        if not column.isidentifier():
            raise DatasetPrepareError(
                name,
                f"import from pandas error - '{column}' cannot be a column name",
            )

    return from_values(
        name,
        session,
        settings=settings,
        object_name=object_name,
        in_memory=in_memory,
        **fr_map,
    )

from_parquet 

from_parquet(
    path,
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs
) -> DataChain

Generate chain from parquet files.

Parameters:

  • path –

    Storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///".

  • partitioning –

    Any pyarrow partitioning schema.

  • output –

    Dictionary defining column names and their corresponding types.

  • object_name –

    Created object column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • session –

    Session to use for the chain.

  • settings –

    Settings to use for the chain.

Example

Reading a single file: 

import datachain as dc
dc.from_parquet("s3://mybucket/file.parquet")

Reading a partitioned dataset from a directory: 

import datachain as dc
dc.from_parquet("s3://mybucket/dir")

Source code in datachain/lib/dc/parquet.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def from_parquet(
    path,
    partitioning: Any = "hive",
    output: Optional[dict[str, DataType]] = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    **kwargs,
) -> "DataChain":
    """Generate chain from parquet files.

    Parameters:
        path : Storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///".
        partitioning : Any pyarrow partitioning schema.
        output : Dictionary defining column names and their corresponding types.
        object_name : Created object column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        session : Session to use for the chain.
        settings : Settings to use for the chain.

    Example:
        Reading a single file:
        ```py
        import datachain as dc
        dc.from_parquet("s3://mybucket/file.parquet")
        ```

        Reading a partitioned dataset from a directory:
        ```py
        import datachain as dc
        dc.from_parquet("s3://mybucket/dir")
        ```
    """
    from .datachain import DataChain

    chain = DataChain.from_storage(path, session=session, settings=settings, **kwargs)
    return chain.parse_tabular(
        output=output,
        object_name=object_name,
        model_name=model_name,
        source=source,
        format="parquet",
        partitioning=partitioning,
    )

from_records 

from_records(
    to_insert: Optional[Union[dict, list[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> DataChain

Create a DataChain from the provided records. This method can be used for programmatically generating a chain in contrast of reading data from storages or other sources.

Parameters:

  • to_insert –

    records (or a single record) to insert. Each record is a dictionary of signals and theirs values.

  • schema –

    describes chain signals and their corresponding types

Example 
import datachain as dc
single_record = dc.from_records(dc.DEFAULT_FILE_RECORD)
Source code in datachain/lib/dc/records.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def from_records(
    to_insert: Optional[Union[dict, list[dict]]],
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    schema: Optional[dict[str, DataType]] = None,
) -> "DataChain":
    """Create a DataChain from the provided records. This method can be used for
    programmatically generating a chain in contrast of reading data from storages
    or other sources.

    Parameters:
        to_insert : records (or a single record) to insert. Each record is
                    a dictionary of signals and theirs values.
        schema : describes chain signals and their corresponding types

    Example:
        ```py
        import datachain as dc
        single_record = dc.from_records(dc.DEFAULT_FILE_RECORD)
        ```
    """
    from .datasets import from_dataset

    session = Session.get(session, in_memory=in_memory)
    catalog = session.catalog

    name = session.generate_temp_dataset_name()
    signal_schema = None
    columns: list[sqlalchemy.Column] = []

    if schema:
        signal_schema = SignalSchema(schema)
        columns = [
            sqlalchemy.Column(c.name, c.type)  # type: ignore[union-attr]
            for c in signal_schema.db_signals(as_columns=True)  # type: ignore[assignment]
        ]
    else:
        columns = [
            sqlalchemy.Column(name, typ)
            for name, typ in File._datachain_column_types.items()
        ]

    dsr = catalog.create_dataset(
        name,
        columns=columns,
        feature_schema=(
            signal_schema.clone_without_sys_signals().serialize()
            if signal_schema
            else None
        ),
    )

    session.add_dataset_version(dsr, dsr.latest_version)

    if isinstance(to_insert, dict):
        to_insert = [to_insert]
    elif not to_insert:
        to_insert = []

    warehouse = catalog.warehouse
    dr = warehouse.dataset_rows(dsr)
    db = warehouse.db
    insert_q = dr.get_table().insert()
    for record in to_insert:
        db.execute(insert_q.values(**record))
    return from_dataset(name=dsr.name, session=session, settings=settings)

from_storage 

from_storage(
    uri: Union[str, PathLike[str]],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    object_name: str = "file",
    update: bool = False,
    anon: bool = False,
    client_config: Optional[dict] = None
) -> DataChain

Get data from a storage as a list of file with all file attributes. It returns the chain itself as usual.

Parameters:

  • uri –

    storage URI with directory. URI must start with storage prefix such as s3://, gs://, az:// or "file:///"

  • type –

    read file as "binary", "text", or "image" data. Default is "binary".

  • recursive –

    search recursively for the given path.

  • object_name –

    Created object column name.

  • update –

    force storage reindexing. Default is False.

  • anon –

    If True, we will treat cloud bucket as public one

  • client_config –

    Optional client configuration for the storage client.

Example

Simple call from s3 

import datachain as dc
chain = dc.from_storage("s3://my-bucket/my-dir")

With AWS S3-compatible storage 

import datachain as dc
chain = dc.from_storage(
    "s3://my-bucket/my-dir",
    client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
)

Pass existing session 

session = Session.get()
import datachain as dc
chain = dc.from_storage("s3://my-bucket/my-dir", session=session)

Source code in datachain/lib/dc/storage.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def from_storage(
    uri: Union[str, os.PathLike[str]],
    *,
    type: FileType = "binary",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    recursive: Optional[bool] = True,
    object_name: str = "file",
    update: bool = False,
    anon: bool = False,
    client_config: Optional[dict] = None,
) -> "DataChain":
    """Get data from a storage as a list of file with all file attributes.
    It returns the chain itself as usual.

    Parameters:
        uri : storage URI with directory. URI must start with storage prefix such
            as `s3://`, `gs://`, `az://` or "file:///"
        type : read file as "binary", "text", or "image" data. Default is "binary".
        recursive : search recursively for the given path.
        object_name : Created object column name.
        update : force storage reindexing. Default is False.
        anon : If True, we will treat cloud bucket as public one
        client_config : Optional client configuration for the storage client.

    Example:
        Simple call from s3
        ```py
        import datachain as dc
        chain = dc.from_storage("s3://my-bucket/my-dir")
        ```

        With AWS S3-compatible storage
        ```py
        import datachain as dc
        chain = dc.from_storage(
            "s3://my-bucket/my-dir",
            client_config = {"aws_endpoint_url": "<minio-endpoint-url>"}
        )
        ```

        Pass existing session
        ```py
        session = Session.get()
        import datachain as dc
        chain = dc.from_storage("s3://my-bucket/my-dir", session=session)
        ```
    """
    from .datachain import DataChain
    from .datasets import from_dataset
    from .records import from_records
    from .values import from_values

    file_type = get_file_type(type)

    if anon:
        client_config = (client_config or {}) | {"anon": True}
    session = Session.get(session, client_config=client_config, in_memory=in_memory)
    cache = session.catalog.cache
    client_config = session.catalog.client_config

    list_ds_name, list_uri, list_path, list_ds_exists = get_listing(
        uri, session, update=update
    )

    # ds_name is None if object is a file, we don't want to use cache
    # or do listing in that case - just read that single object
    if not list_ds_name:
        dc = from_values(
            session=session,
            settings=settings,
            in_memory=in_memory,
            file=[get_file_info(list_uri, cache, client_config=client_config)],
        )
        dc.signals_schema = dc.signals_schema.mutate({f"{object_name}": file_type})
        return dc

    if update or not list_ds_exists:
        # disable prefetch for listing, as it pre-downloads all files
        (
            from_records(
                DataChain.DEFAULT_FILE_RECORD,
                session=session,
                settings=settings,
                in_memory=in_memory,
            )
            .settings(prefetch=0)
            .gen(
                list_bucket(list_uri, cache, client_config=client_config),
                output={f"{object_name}": File},
            )
            .save(list_ds_name, listing=True)
        )

    dc = from_dataset(list_ds_name, session=session, settings=settings)
    dc.signals_schema = dc.signals_schema.mutate({f"{object_name}": file_type})

    return ls(dc, list_path, recursive=recursive, object_name=object_name)

from_values 

from_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    object_name: str = "",
    **fr_map
) -> DataChain

Generate chain from list of values.

Example 
import datachain as dc
dc.from_values(fib=[1, 2, 3, 5, 8])
Source code in datachain/lib/dc/values.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
def from_values(
    ds_name: str = "",
    session: Optional[Session] = None,
    settings: Optional[dict] = None,
    in_memory: bool = False,
    output: OutputType = None,
    object_name: str = "",
    **fr_map,
) -> "DataChain":
    """Generate chain from list of values.

    Example:
        ```py
        import datachain as dc
        dc.from_values(fib=[1, 2, 3, 5, 8])
        ```
    """
    from .datachain import DataChain

    tuple_type, output, tuples = values_to_tuples(ds_name, output, **fr_map)

    def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
        yield from tuples

    chain = from_records(
        DataChain.DEFAULT_FILE_RECORD,
        session=session,
        settings=settings,
        in_memory=in_memory,
    )
    if object_name:
        output = {object_name: dict_to_data_model(object_name, output)}  # type: ignore[arg-type]
    return chain.gen(_func_fr, output=output)

DataChain 

DataChain(
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
)

DataChain - a data structure for batch data processing and evaluation.

It represents a sequence of data manipulation steps such as reading data from storages, running AI or LLM models or calling external services API to validate or enrich data.

Data in DataChain is presented as Python classes with arbitrary set of fields, including nested classes. The data classes have to inherit from DataModel class. The supported set of field types include: majority of the type supported by the underlyind library Pydantic.

See Also

from_storage("s3://my-bucket/my-dir/") - reading unstructured data files from storages such as S3, gs or Azure ADLS.

DataChain.save("name") - saving to a dataset.

from_dataset("name") - reading from a dataset.

from_values(fib=[1, 2, 3, 5, 8]) - generating from values.

from_pandas(pd.DataFrame(...)) - generating from pandas.

from_json("file.json") - generating from json.

from_csv("file.csv") - generating from csv.

from_parquet("file.parquet") - generating from parquet.

Example 
import os

from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage
import datachain as dc

PROMPT = (
    "Was this bot dialog successful? "
    "Describe the 'result' as 'Yes' or 'No' in a short JSON"
)

model = "mistral-large-latest"
api_key = os.environ["MISTRAL_API_KEY"]

chain = (
    dc.from_storage("gs://datachain-demo/chatbot-KiT/")
    .limit(5)
    .settings(cache=True, parallel=5)
    .map(
        mistral_response=lambda file: MistralClient(api_key=api_key)
        .chat(
            model=model,
            response_format={"type": "json_object"},
            messages=[
                ChatMessage(role="user", content=f"{PROMPT}: {file.read()}")
            ],
        )
        .choices[0]
        .message.content,
    )
    .save()
)

try:
    print(chain.select("mistral_response").results())
except Exception as e:
    print(f"do you have the right Mistral API key? {e}")
Source code in datachain/lib/dc/datachain.py 
152
153
154
155
156
157
158
159
160
161
162
163
164
165
def __init__(
    self,
    query: DatasetQuery,
    settings: Settings,
    signal_schema: SignalSchema,
    setup: Optional[dict] = None,
    _sys: bool = False,
) -> None:
    """Don't instantiate this directly, use one of the from_XXX constructors."""
    self._query = query
    self._settings = settings
    self.signals_schema = signal_schema
    self._setup: dict = setup or {}
    self._sys = _sys

dataset property 

dataset: Optional[DatasetRecord]

Underlying dataset, if there is one.

name property 

name: Optional[str]

Name of the underlying dataset, if there is one.

schema property 

schema: dict[str, DataType]

Get schema of the chain.

session property 

session: Session

Session of the chain.

version property 

version: Optional[int]

Version of the underlying dataset, if there is one.

__or__ 

__or__(other: Self) -> Self

Return self.union(other).

Source code in datachain/lib/dc/datachain.py 
228
229
230
def __or__(self, other: "Self") -> "Self":
    """Return `self.union(other)`."""
    return self.union(other)

__repr__ 

__repr__() -> str

Return a string representation of the chain.

Source code in datachain/lib/dc/datachain.py 
167
168
169
170
171
172
173
174
175
176
177
def __repr__(self) -> str:
    """Return a string representation of the chain."""
    classname = self.__class__.__name__
    if not self._effective_signals_schema.values:
        return f"Empty {classname}"

    import io

    file = io.StringIO()
    self.print_schema(file=file)
    return file.getvalue()

agg 

agg(
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Aggregate rows using partition_by statement and apply a function to the groups of aggregated rows. The function needs to return new objects for each group of the new rows. It returns a chain itself with new signals.

Input-output relationship: N:M

This method bears similarity to gen() and map(), employing a comparable set of parameters, yet differs in two crucial aspects: 1. The partition_by parameter: This specifies the column name or a list of column names that determine the grouping criteria for aggregation. 2. Group-based UDF function input: Instead of individual rows, the function receives a list all rows within each group defined by partition_by.

Examples:

chain = chain.agg(
    total=lambda category, amount: [sum(amount)],
    output=float,
    partition_by="category",
)
chain.save("new_dataset")

An alternative syntax, when you need to specify a more complex function:

# It automatically resolves which columns to pass to the function
# by looking at the function signature.
def agg_sum(
    file: list[File], amount: list[float]
) -> Iterator[tuple[File, float]]:
    yield file[0], sum(amount)

chain = chain.agg(
    agg_sum,
    output={"file": File, "total": float},
    # Alternative syntax is to use `C` (short for Column) to specify
    # a column name or a nested column, e.g. C("file.path").
    partition_by=C("category"),
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
def agg(
    self,
    func: Optional[Callable] = None,
    partition_by: Optional[PartitionByType] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    """Aggregate rows using `partition_by` statement and apply a function to the
    groups of aggregated rows. The function needs to return new objects for each
    group of the new rows. It returns a chain itself with new signals.

    Input-output relationship: N:M

    This method bears similarity to `gen()` and `map()`, employing a comparable set
    of parameters, yet differs in two crucial aspects:
    1. The `partition_by` parameter: This specifies the column name or a list of
       column names that determine the grouping criteria for aggregation.
    2. Group-based UDF function input: Instead of individual rows, the function
       receives a list all rows within each group defined by `partition_by`.

    Examples:
        ```py
        chain = chain.agg(
            total=lambda category, amount: [sum(amount)],
            output=float,
            partition_by="category",
        )
        chain.save("new_dataset")
        ```

        An alternative syntax, when you need to specify a more complex function:

        ```py
        # It automatically resolves which columns to pass to the function
        # by looking at the function signature.
        def agg_sum(
            file: list[File], amount: list[float]
        ) -> Iterator[tuple[File, float]]:
            yield file[0], sum(amount)

        chain = chain.agg(
            agg_sum,
            output={"file": File, "total": float},
            # Alternative syntax is to use `C` (short for Column) to specify
            # a column name or a nested column, e.g. C("file.path").
            partition_by=C("category"),
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Aggregator, func, params, output, signal_map)
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(),
            partition_by=partition_by,
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

apply 

apply(func, *args, **kwargs)

Apply any function to the chain.

Useful for reusing in a chain of operations.

Example 
import datachain as dc
def parse_stem(chain):
    return chain.map(
        lambda file: file.get_file_stem()
        output={"stem": str}
    )

chain = (
    dc.from_storage("s3://my-bucket")
    .apply(parse_stem)
    .filter(C("stem").glob("*cat*"))
)
Source code in datachain/lib/dc/datachain.py 
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
def apply(self, func, *args, **kwargs):
    """Apply any function to the chain.

    Useful for reusing in a chain of operations.

    Example:
        ```py
        import datachain as dc
        def parse_stem(chain):
            return chain.map(
                lambda file: file.get_file_stem()
                output={"stem": str}
            )

        chain = (
            dc.from_storage("s3://my-bucket")
            .apply(parse_stem)
            .filter(C("stem").glob("*cat*"))
        )
        ```
    """
    return func(self, *args, **kwargs)

avg 

avg(fr: DataType)

Compute the average of a column.

Source code in datachain/lib/dc/datachain.py 
1944
1945
1946
def avg(self, fr: DataType):  # type: ignore[override]
    """Compute the average of a column."""
    return self._extend_to_data_model("avg", fr)

batch_map 

batch_map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map
) -> Self

This is a batch version of map().

Input-output relationship: N:N

It accepts the same parameters plus an additional parameter:

batch : Size of each batch passed to `func`. Defaults to 1000.
Example 
chain = chain.batch_map(
    sqrt=lambda size: np.sqrt(size),
    output=float
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
def batch_map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    batch: int = 1000,
    **signal_map,
) -> "Self":
    """This is a batch version of `map()`.

    Input-output relationship: N:N

    It accepts the same parameters plus an
    additional parameter:

        batch : Size of each batch passed to `func`. Defaults to 1000.

    Example:
        ```py
        chain = chain.batch_map(
            sqrt=lambda size: np.sqrt(size),
            output=float
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(BatchMapper, func, params, output, signal_map)
    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(batch),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

c 

c(column: Union[str, Column]) -> Column

Returns Column instance attached to the current chain.

Source code in datachain/lib/dc/datachain.py 
200
201
202
203
204
def c(self, column: Union[str, Column]) -> Column:
    """Returns Column instance attached to the current chain."""
    c = self.column(column) if isinstance(column, str) else self.column(column.name)
    c.table = self._query.table
    return c

chunk 

chunk(index: int, total: int) -> Self

Split a chain into smaller chunks for e.g. parallelization.

Example 
chain = DataChain.from_storage(...)
chunk_1 = query._chunk(0, 2)
chunk_2 = query._chunk(1, 2)
Note

Bear in mind that index is 0-indexed but total isn't. Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.

Source code in datachain/lib/dc/datachain.py 
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
def chunk(self, index: int, total: int) -> "Self":
    """Split a chain into smaller chunks for e.g. parallelization.

    Example:
        ```py
        chain = DataChain.from_storage(...)
        chunk_1 = query._chunk(0, 2)
        chunk_2 = query._chunk(1, 2)
        ```

    Note:
        Bear in mind that `index` is 0-indexed but `total` isn't.
        Use 0/3, 1/3 and 2/3, not 1/3, 2/3 and 3/3.
    """
    return self._evolve(query=self._query.chunk(index, total))

clone 

clone() -> Self

Make a copy of the chain in a new table.

Source code in datachain/lib/dc/datachain.py 
236
237
238
def clone(self) -> "Self":
    """Make a copy of the chain in a new table."""
    return self._evolve(query=self._query.clone(new_table=True))

collect 

collect() -> Iterator[tuple[DataValue, ...]]

collect(col: str) -> Iterator[DataValue]

collect(*cols: str) -> Iterator[tuple[DataValue, ...]]

collect(
    *cols: str,
) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]

Yields rows of values, optionally limited to the specified columns.

Parameters:

  • *cols (str, default: () ) –

    Limit to the specified columns. By default, all columns are selected.

Yields:

  • DataType –

    Yields a single item if a column is selected.

  • tuple[DataType, ...] –

    Yields a tuple of items if multiple columns are selected.

Example

Iterating over all rows: 

for row in dc.collect():
    print(row)

Iterating over all rows with selected columns: 

for name, size in dc.collect("file.name", "file.size"):
    print(name, size)

Iterating over a single column: 

for file in dc.collect("file.name"):
    print(file)

Source code in datachain/lib/dc/datachain.py 
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
def collect(self, *cols: str) -> Iterator[Union[DataValue, tuple[DataValue, ...]]]:  # type: ignore[overload-overlap,misc]
    """Yields rows of values, optionally limited to the specified columns.

    Args:
        *cols: Limit to the specified columns. By default, all columns are selected.

    Yields:
        (DataType): Yields a single item if a column is selected.
        (tuple[DataType, ...]): Yields a tuple of items if multiple columns are
            selected.

    Example:
        Iterating over all rows:
        ```py
        for row in dc.collect():
            print(row)
        ```

        Iterating over all rows with selected columns:
        ```py
        for name, size in dc.collect("file.name", "file.size"):
            print(name, size)
        ```

        Iterating over a single column:
        ```py
        for file in dc.collect("file.name"):
            print(file)
        ```
    """
    chain = self.select(*cols) if cols else self
    signals_schema = chain._effective_signals_schema
    db_signals = signals_schema.db_signals()
    with self._query.ordered_select(*db_signals).as_iterable() as rows:
        for row in rows:
            ret = signals_schema.row_to_features(
                row, catalog=chain.session.catalog, cache=chain._settings.cache
            )
            yield ret[0] if len(cols) == 1 else tuple(ret)

collect_flatten 

collect_flatten() -> Iterator[tuple[Any, ...]]

collect_flatten(
    *, include_hidden: bool
) -> Iterator[tuple[Any, ...]]

collect_flatten(
    *,
    row_factory: Callable[[list[str], tuple[Any, ...]], _T]
) -> Iterator[_T]

collect_flatten(
    *,
    row_factory: Callable[[list[str], tuple[Any, ...]], _T],
    include_hidden: bool
) -> Iterator[_T]

collect_flatten(
    *, row_factory=None, include_hidden: bool = True
)

Yields flattened rows of values as a tuple.

Parameters:

  • row_factory –

    A callable to convert row to a custom format. It should accept two arguments: a list of column names and a tuple of row values.

  • include_hidden (bool, default: True ) –

    Whether to include hidden signals from the schema.

Source code in datachain/lib/dc/datachain.py 
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
def collect_flatten(self, *, row_factory=None, include_hidden: bool = True):
    """Yields flattened rows of values as a tuple.

    Args:
        row_factory : A callable to convert row to a custom format.
                      It should accept two arguments: a list of column names and
                      a tuple of row values.
        include_hidden: Whether to include hidden signals from the schema.
    """
    db_signals = self._effective_signals_schema.db_signals(
        include_hidden=include_hidden
    )
    with self._query.ordered_select(*db_signals).as_iterable() as rows:
        if row_factory:
            rows = (row_factory(db_signals, r) for r in rows)  # type: ignore[assignment]
        yield from rows

column 

column(name: str) -> Column

Returns Column instance with a type if name is found in current schema, otherwise raises an exception.

Source code in datachain/lib/dc/datachain.py 
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
def column(self, name: str) -> Column:
    """Returns Column instance with a type if name is found in current schema,
    otherwise raises an exception.
    """
    if "." in name:
        name_path = name.split(".")
    elif DEFAULT_DELIMITER in name:
        name_path = name.split(DEFAULT_DELIMITER)
    else:
        name_path = [name]
    for path, type_, _, _ in self.signals_schema.get_flat_tree():
        if path == name_path:
            return Column(name, python_to_sql(type_))

    raise ValueError(f"Column with name {name} not found in the schema")

compare 

compare(
    other: DataChain,
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[
        Union[str, Sequence[str]]
    ] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Comparing two chains by identifying rows that are added, deleted, modified or same. Result is the new chain that has additional column with possible values: A, D, M, U representing added, deleted, modified and same rows respectively. Note that if only one "status" is asked, by setting proper flags, this additional column is not created as it would have only one value for all rows. Beside additional diff column, new chain has schema of the chain on which method was called.

Parameters:

  • other (DataChain) –

    Chain to calculate diff from.

  • on (Union[str, Sequence[str]]) –

    Column or list of columns to match on. If both chains have the same columns then this column is enough for the match. Otherwise, right_on parameter has to specify the columns for the other chain. This value is used to find corresponding row in other dataset. If not found there, row is considered as added (or removed if vice versa), and if found then row can be either modified or same.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    Optional column or list of columns for the other to match.

  • compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Column or list of columns to compare on. If both chains have the same columns then this column is enough for the compare. Otherwise, right_compare parameter has to specify the columns for the other chain. This value is used to see if row is modified or same. If not set, all columns will be used for comparison

  • right_compare (Optional[Union[str, Sequence[str]]], default: None ) –

    Optional column or list of columns for the other to compare to.

  • added (bool, default: True ) –

    Whether to return added rows in resulting chain.

  • deleted (bool, default: True ) –

    Whether to return deleted rows in resulting chain.

  • modified (bool, default: True ) –

    Whether to return modified rows in resulting chain.

  • same (bool, default: False ) –

    Whether to return unchanged rows in resulting chain.

  • status_col (str, default: None ) –

    Name of the new column that is created in resulting chain representing diff status.

Example 
res = persons.compare(
    new_persons,
    on=["id"],
    right_on=["other_id"],
    compare=["name"],
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
def compare(
    self,
    other: "DataChain",
    on: Union[str, Sequence[str]],
    right_on: Optional[Union[str, Sequence[str]]] = None,
    compare: Optional[Union[str, Sequence[str]]] = None,
    right_compare: Optional[Union[str, Sequence[str]]] = None,
    added: bool = True,
    deleted: bool = True,
    modified: bool = True,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Comparing two chains by identifying rows that are added, deleted, modified
    or same. Result is the new chain that has additional column with possible
    values: `A`, `D`, `M`, `U` representing added, deleted, modified and same
    rows respectively. Note that if only one "status" is asked, by setting proper
    flags, this additional column is not created as it would have only one value
    for all rows. Beside additional diff column, new chain has schema of the chain
    on which method was called.

    Parameters:
        other: Chain to calculate diff from.
        on: Column or list of columns to match on. If both chains have the
            same columns then this column is enough for the match. Otherwise,
            `right_on` parameter has to specify the columns for the other chain.
            This value is used to find corresponding row in other dataset. If not
            found there, row is considered as added (or removed if vice versa), and
            if found then row can be either modified or same.
        right_on: Optional column or list of columns
            for the `other` to match.
        compare: Column or list of columns to compare on. If both chains have
            the same columns then this column is enough for the compare. Otherwise,
            `right_compare` parameter has to specify the columns for the other
            chain. This value is used to see if row is modified or same. If
            not set, all columns will be used for comparison
        right_compare: Optional column or list of columns
                for the `other` to compare to.
        added (bool): Whether to return added rows in resulting chain.
        deleted (bool): Whether to return deleted rows in resulting chain.
        modified (bool): Whether to return modified rows in resulting chain.
        same (bool): Whether to return unchanged rows in resulting chain.
        status_col (str): Name of the new column that is created in resulting chain
            representing diff status.

    Example:
        ```py
        res = persons.compare(
            new_persons,
            on=["id"],
            right_on=["other_id"],
            compare=["name"],
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    from datachain.diff import _compare

    return _compare(
        self,
        other,
        on,
        right_on=right_on,
        compare=compare,
        right_compare=right_compare,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

count 

count() -> int

Return the number of rows in the chain.

Source code in datachain/lib/dc/datachain.py 
2121
2122
2123
def count(self) -> int:
    """Return the number of rows in the chain."""
    return self._query.count()

diff 

diff(
    other: DataChain,
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> DataChain

Similar to .compare(), which is more generic method to calculate difference between two chains. Unlike .compare(), this method works only on those chains that have File object, or it's derivatives, in it. File source and path are used for matching, and file version and etag for comparing, while in .compare() user needs to provide arbitrary columns for matching and comparing.

Parameters:

  • other (DataChain) –

    Chain to calculate diff from.

  • on (str, default: 'file' ) –

    File signal to match on. If both chains have the same file signal then this column is enough for the match. Otherwise, right_on parameter has to specify the file signal for the other chain. This value is used to find corresponding row in other dataset. If not found there, row is considered as added (or removed if vice versa), and if found then row can be either modified or same.

  • right_on (Optional[str], default: None ) –

    Optional file signal for the other to match.

  • added (bool, default: True ) –

    Whether to return added rows in resulting chain.

  • deleted (bool, default: False ) –

    Whether to return deleted rows in resulting chain.

  • modified (bool, default: True ) –

    Whether to return modified rows in resulting chain.

  • same (bool, default: False ) –

    Whether to return unchanged rows in resulting chain.

  • status_col (str, default: None ) –

    Optional name of the new column that is created in resulting chain representing diff status.

Example 
diff = images.diff(
    new_images,
    on="file",
    right_on="other_file",
    added=True,
    deleted=True,
    modified=True,
    same=True,
    status_col="diff"
)
Source code in datachain/lib/dc/datachain.py 
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
def diff(
    self,
    other: "DataChain",
    on: str = "file",
    right_on: Optional[str] = None,
    added: bool = True,
    modified: bool = True,
    deleted: bool = False,
    same: bool = False,
    status_col: Optional[str] = None,
) -> "DataChain":
    """Similar to `.compare()`, which is more generic method to calculate difference
    between two chains. Unlike `.compare()`, this method works only on those chains
    that have `File` object, or it's derivatives, in it. File `source` and `path`
    are used for matching, and file `version` and `etag` for comparing, while in
    `.compare()` user needs to provide arbitrary columns for matching and comparing.

    Parameters:
        other: Chain to calculate diff from.
        on: File signal to match on. If both chains have the
            same file signal then this column is enough for the match. Otherwise,
            `right_on` parameter has to specify the file signal for the other chain.
            This value is used to find corresponding row in other dataset. If not
            found there, row is considered as added (or removed if vice versa), and
            if found then row can be either modified or same.
        right_on: Optional file signal for the `other` to match.
        added (bool): Whether to return added rows in resulting chain.
        deleted (bool): Whether to return deleted rows in resulting chain.
        modified (bool): Whether to return modified rows in resulting chain.
        same (bool): Whether to return unchanged rows in resulting chain.
        status_col (str): Optional name of the new column that is created in
            resulting chain representing diff status.

    Example:
        ```py
        diff = images.diff(
            new_images,
            on="file",
            right_on="other_file",
            added=True,
            deleted=True,
            modified=True,
            same=True,
            status_col="diff"
        )
        ```
    """
    on_file_signals = ["source", "path"]
    compare_file_signals = ["version", "etag"]

    def get_file_signals(file: str, signals):
        return [f"{file}.{c}" for c in signals]

    right_on = right_on or on

    on_cols = get_file_signals(on, on_file_signals)
    right_on_cols = get_file_signals(right_on, on_file_signals)
    compare_cols = get_file_signals(on, compare_file_signals)
    right_compare_cols = get_file_signals(right_on, compare_file_signals)

    return self.compare(
        other,
        on_cols,
        right_on=right_on_cols,
        compare=compare_cols,
        right_compare=right_compare_cols,
        added=added,
        deleted=deleted,
        modified=modified,
        same=same,
        status_col=status_col,
    )

distinct 

distinct(arg: str, *args: str) -> Self

Removes duplicate rows based on uniqueness of some input column(s) i.e if rows are found with the same value of input column(s), only one row is left in the result set.

Example 
dc.distinct("file.parent", "file.name")
Source code in datachain/lib/dc/datachain.py 
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
def distinct(self, arg: str, *args: str) -> "Self":  # type: ignore[override]
    """Removes duplicate rows based on uniqueness of some input column(s)
    i.e if rows are found with the same value of input column(s), only one
    row is left in the result set.

    Example:
        ```py
        dc.distinct("file.parent", "file.name")
        ```
    """
    return self._evolve(
        query=self._query.distinct(
            *self.signals_schema.resolve(arg, *args).db_signals()
        )
    )

exec 

exec() -> Self

Execute the chain.

Source code in datachain/lib/dc/datachain.py 
2125
2126
2127
def exec(self) -> "Self":
    """Execute the chain."""
    return self._evolve(query=self._query.exec())

explode 

explode(
    col: str,
    model_name: Optional[str] = None,
    object_name: Optional[str] = None,
    schema_sample_size: int = 1,
) -> DataChain

Explodes a column containing JSON objects (dict or str DataChain type) into individual columns based on the schema of the JSON. Schema is inferred from the first row of the column.

Parameters:

  • col (str) –

    the name of the column containing JSON to be exploded.

  • model_name (Optional[str], default: None ) –

    optional generated model name. By default generates the name automatically.

  • object_name (Optional[str], default: None ) –

    optional generated object column name. By default generates the name automatically.

  • schema_sample_size (int, default: 1 ) –

    the number of rows to use for inferring the schema of the JSON (in case some fields are optional and it's not enough to analyze a single row).

Returns:

  • DataChain ( DataChain ) –

    A new DataChain instance with the new set of columns.

Source code in datachain/lib/dc/datachain.py 
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
def explode(
    self,
    col: str,
    model_name: Optional[str] = None,
    object_name: Optional[str] = None,
    schema_sample_size: int = 1,
) -> "DataChain":
    """Explodes a column containing JSON objects (dict or str DataChain type) into
       individual columns based on the schema of the JSON. Schema is inferred from
       the first row of the column.

    Args:
        col: the name of the column containing JSON to be exploded.
        model_name: optional generated model name.  By default generates the name
            automatically.
        object_name: optional generated object column name. By default generates the
            name automatically.
        schema_sample_size: the number of rows to use for inferring the schema of
            the JSON (in case some fields are optional and it's not enough to
            analyze a single row).

    Returns:
        DataChain: A new DataChain instance with the new set of columns.
    """
    import json

    import pyarrow as pa

    from datachain.lib.arrow import schema_to_output

    json_values = list(self.limit(schema_sample_size).collect(col))
    json_dicts = [
        json.loads(json_value) if isinstance(json_value, str) else json_value
        for json_value in json_values
    ]

    if any(not isinstance(json_dict, dict) for json_dict in json_dicts):
        raise TypeError(f"Column {col} should be a string or dict type with JSON")

    schema = pa.Table.from_pylist(json_dicts).schema
    output, original_names = schema_to_output(schema, None)

    if not model_name:
        model_name = f"{col.title()}ExplodedModel"

    model = dict_to_data_model(model_name, output, original_names)

    def json_to_model(json_value: Union[str, dict]):
        json_dict = (
            json.loads(json_value) if isinstance(json_value, str) else json_value
        )
        return model.model_validate(json_dict)

    if not object_name:
        object_name = f"{col}_expl"

    return self.map(json_to_model, params=col, output={object_name: model})

filter 

filter(*args: Any) -> Self

Filter the chain according to conditions.

Example

Basic usage with built-in operators 

dc.filter(C("width") < 200)

Using glob to match patterns 

dc.filter(C("file.name").glob("*.jpg"))

Using datachain.func 

from datachain.func import string
dc.filter(string.length(C("file.name")) > 5)

Combining filters with "or" 

dc.filter(C("file.name").glob("cat*") | C("file.name").glob("dog*))

Combining filters with "and" 

dc.filter(
    C("file.name").glob("*.jpg) &
    (string.length(C("file.name")) > 5)
)

Source code in datachain/lib/dc/datachain.py 
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
def filter(self, *args: Any) -> "Self":
    """Filter the chain according to conditions.

    Example:
        Basic usage with built-in operators
        ```py
        dc.filter(C("width") < 200)
        ```

        Using glob to match patterns
        ```py
        dc.filter(C("file.name").glob("*.jpg"))
        ```

        Using `datachain.func`
        ```py
        from datachain.func import string
        dc.filter(string.length(C("file.name")) > 5)
        ```

        Combining filters with "or"
        ```py
        dc.filter(C("file.name").glob("cat*") | C("file.name").glob("dog*))
        ```

        Combining filters with "and"
        ```py
        dc.filter(
            C("file.name").glob("*.jpg) &
            (string.length(C("file.name")) > 5)
        )
        ```
    """
    return self._evolve(query=self._query.filter(*args))

gen 

gen(
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new rows (with potentially new signals). The function needs to return a new objects for each of the new rows. It returns a chain itself with new signals.

Input-output relationship: 1:N

This method is similar to map(), uses the same list of parameters, but with one key differences: It produces a sequence of rows for each input row (like extracting multiple file records from a single tar file or bounding boxes from a single image file).

Example 
chain = chain.gen(
    line=lambda file: [l for l in file.read().split("\n")],
    output=str,
)
chain.save("new_dataset")
Source code in datachain/lib/dc/datachain.py 
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
def gen(
    self,
    func: Optional[Union[Callable, Generator]] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    r"""Apply a function to each row to create new rows (with potentially new
    signals). The function needs to return a new objects for each of the new rows.
    It returns a chain itself with new signals.

    Input-output relationship: 1:N

    This method is similar to `map()`, uses the same list of parameters, but with
    one key differences: It produces a sequence of rows for each input row (like
    extracting multiple file records from a single tar file or bounding boxes from a
    single image file).

    Example:
        ```py
        chain = chain.gen(
            line=lambda file: [l for l in file.read().split("\n")],
            output=str,
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Generator, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch
    return self._evolve(
        query=self._query.generate(
            udf_obj.to_udf_wrapper(),
            **self._settings.to_dict(),
        ),
        signal_schema=udf_obj.output,
    )

group_by 

group_by(
    *,
    partition_by: Optional[
        Union[str, Func, Sequence[Union[str, Func]]]
    ] = None,
    **kwargs: Func
) -> Self

Group rows by specified set of signals and return new signals with aggregated values.

The supported functions

count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

Example 
chain = chain.group_by(
    cnt=func.count(),
    partition_by=("file_source", "file_ext"),
)
Source code in datachain/lib/dc/datachain.py 
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
def group_by(
    self,
    *,
    partition_by: Optional[Union[str, Func, Sequence[Union[str, Func]]]] = None,
    **kwargs: Func,
) -> "Self":
    """Group rows by specified set of signals and return new signals
    with aggregated values.

    The supported functions:
       count(), sum(), avg(), min(), max(), any_value(), collect(), concat()

    Example:
        ```py
        chain = chain.group_by(
            cnt=func.count(),
            partition_by=("file_source", "file_ext"),
        )
        ```
    """
    if partition_by is None:
        partition_by = []
    elif isinstance(partition_by, (str, Func)):
        partition_by = [partition_by]

    partition_by_columns: list[Column] = []
    signal_columns: list[Column] = []
    schema_fields: dict[str, DataType] = {}
    keep_columns: list[str] = []

    # validate partition_by columns and add them to the schema
    for col in partition_by:
        if isinstance(col, str):
            col_db_name = ColumnMeta.to_db_name(col)
            col_type = self.signals_schema.get_column_type(col_db_name)
            column = Column(col_db_name, python_to_sql(col_type))
            if col not in keep_columns:
                keep_columns.append(col)
        elif isinstance(col, Function):
            column = col.get_column(self.signals_schema)
            col_db_name = column.name
            col_type = column.type.python_type
            schema_fields[col_db_name] = col_type
        else:
            raise DataChainColumnError(
                col,
                (
                    f"partition_by column {col} has type {type(col)}"
                    " but expected str or Function"
                ),
            )
        partition_by_columns.append(column)

    # validate signal columns and add them to the schema
    if not kwargs:
        raise ValueError("At least one column should be provided for group_by")
    for col_name, func in kwargs.items():
        if not isinstance(func, Func):
            raise DataChainColumnError(
                col_name,
                f"Column {col_name} has type {type(func)} but expected Func object",
            )
        column = func.get_column(self.signals_schema, label=col_name)
        signal_columns.append(column)
        schema_fields[col_name] = func.get_result_type(self.signals_schema)

    signal_schema = SignalSchema(schema_fields)
    if keep_columns:
        signal_schema |= self.signals_schema.to_partial(*keep_columns)

    return self._evolve(
        query=self._query.group_by(signal_columns, partition_by_columns),
        signal_schema=signal_schema,
    )

limit 

limit(n: int) -> Self

Return the first n rows of the chain.

If the chain is unordered, which rows are returned is undefined. If the chain has less than n rows, the whole chain is returned.

Parameters:

  • n (int) –

    Number of rows to return.

Source code in datachain/lib/dc/datachain.py 
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
def limit(self, n: int) -> "Self":
    """Return the first `n` rows of the chain.

    If the chain is unordered, which rows are returned is undefined.
    If the chain has less than `n` rows, the whole chain is returned.

    Parameters:
        n (int): Number of rows to return.
    """
    return self._evolve(query=self._query.limit(n))

map 

map(
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map
) -> Self

Apply a function to each row to create new signals. The function should return a new object for each row. It returns a chain itself with new signals.

Input-output relationship: 1:1

Parameters:

  • func –

    Function applied to each row.

  • params –

    List of column names used as input for the function. Default is taken from function signature.

  • output –

    Dictionary defining new signals and their corresponding types. Default type is taken from function signature. Default can be also taken from kwargs - **signal_map (see below). If signal name is defined using signal_map (see below) only a single type value can be used.

  • **signal_map –

    kwargs can be used to define func together with it's return signal name in format of map(my_sign=my_func). This helps define signal names and function in a nicer way.

Example

Using signal_map and single type in output: 

chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
chain.save("new_dataset")

Using func and output as a map: 

chain = chain.map(
    lambda name: name.split("."), output={"stem": str, "ext": str}
)
chain.save("new_dataset")

Source code in datachain/lib/dc/datachain.py 
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
def map(
    self,
    func: Optional[Callable] = None,
    params: Union[None, str, Sequence[str]] = None,
    output: OutputType = None,
    **signal_map,
) -> "Self":
    """Apply a function to each row to create new signals. The function should
    return a new object for each row. It returns a chain itself with new signals.

    Input-output relationship: 1:1

    Parameters:
        func : Function applied to each row.
        params : List of column names used as input for the function. Default
                is taken from function signature.
        output : Dictionary defining new signals and their corresponding types.
                Default type is taken from function signature. Default can be also
                taken from kwargs - **signal_map (see below).
                If signal name is defined using signal_map (see below) only a single
                type value can be used.
        **signal_map : kwargs can be used to define `func` together with it's return
                signal name in format of `map(my_sign=my_func)`. This helps define
                signal names and function in a nicer way.

    Example:
        Using signal_map and single type in output:
        ```py
        chain = chain.map(value=lambda name: name[:-4] + ".json", output=str)
        chain.save("new_dataset")
        ```

        Using func and output as a map:
        ```py
        chain = chain.map(
            lambda name: name.split("."), output={"stem": str, "ext": str}
        )
        chain.save("new_dataset")
        ```
    """
    udf_obj = self._udf_to_obj(Mapper, func, params, output, signal_map)
    if (prefetch := self._settings.prefetch) is not None:
        udf_obj.prefetch = prefetch

    return self._evolve(
        query=self._query.add_signals(
            udf_obj.to_udf_wrapper(),
            **self._settings.to_dict(),
        ),
        signal_schema=self.signals_schema | udf_obj.output,
    )

max 

max(fr: DataType)

Compute the maximum of a column.

Source code in datachain/lib/dc/datachain.py 
1952
1953
1954
def max(self, fr: DataType):  # type: ignore[override]
    """Compute the maximum of a column."""
    return self._extend_to_data_model("max", fr)

merge 

merge(
    right_ds: DataChain,
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[
        Union[MergeColType, Sequence[MergeColType]]
    ] = None,
    inner=False,
    full=False,
    rname="right_",
) -> Self

Merge two chains based on the specified criteria.

Parameters:

  • right_ds (DataChain) –

    Chain to join with.

  • on (Union[MergeColType, Sequence[MergeColType]]) –

    Predicate ("column.name", C("column.name"), or Func) or list of Predicates to join on. If both chains have the same predicates then this predicate is enough for the join. Otherwise, right_on parameter has to specify the predicates for the other chain.

  • right_on (Optional[Union[MergeColType, Sequence[MergeColType]]], default: None ) –

    Optional predicate or list of Predicates for the right_ds to join.

  • inner (bool, default: False ) –

    Whether to run inner join or outer join.

  • full (bool, default: False ) –

    Whether to run full outer join.

  • rname (str, default: 'right_' ) –

    Name prefix for conflicting signal names.

Examples:

meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                      right_on=(C.name, C.pq__index))

imgs.merge(captions,
           on=func.path.file_stem(imgs.c("file.path")),
           right_on=func.path.file_stem(captions.c("file.path"))

)

Source code in datachain/lib/dc/datachain.py 
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
def merge(
    self,
    right_ds: "DataChain",
    on: Union[MergeColType, Sequence[MergeColType]],
    right_on: Optional[Union[MergeColType, Sequence[MergeColType]]] = None,
    inner=False,
    full=False,
    rname="right_",
) -> "Self":
    """Merge two chains based on the specified criteria.

    Parameters:
        right_ds: Chain to join with.
        on: Predicate ("column.name", C("column.name"), or Func) or list of
            Predicates to join on. If both chains have the same predicates then
            this predicate is enough for the join. Otherwise, `right_on` parameter
            has to specify the predicates for the other chain.
        right_on: Optional predicate or list of Predicates for the `right_ds`
            to join.
        inner (bool): Whether to run inner join or outer join.
        full (bool): Whether to run full outer join.
        rname (str): Name prefix for conflicting signal names.

    Examples:
        ```py
        meta = meta_emd.merge(meta_pq, on=(C.name, C.emd__index),
                              right_on=(C.name, C.pq__index))
        ```

        ```py
        imgs.merge(captions,
                   on=func.path.file_stem(imgs.c("file.path")),
                   right_on=func.path.file_stem(captions.c("file.path"))
        ```
    )
    """
    if on is None:
        raise DatasetMergeError(["None"], None, "'on' must be specified")

    on = _validate_merge_on(on, self)
    if not on:
        raise DatasetMergeError(
            on,
            right_on,
            (
                "'on' must be 'str', 'Func' or 'Sequence' object "
                f"but got type '{type(on)}'"
            ),
        )

    if right_on is not None:
        right_on = _validate_merge_on(right_on, right_ds)
        if not right_on:
            raise DatasetMergeError(
                on,
                right_on,
                "'right_on' must be 'str', 'Func' or 'Sequence' object"
                f" but got type '{type(right_on)}'",
            )

        if len(right_on) != len(on):
            raise DatasetMergeError(
                on, right_on, "'on' and 'right_on' must have the same length'"
            )

    if self == right_ds:
        right_ds = right_ds.clone()

    errors = []

    def _resolve(
        ds: DataChain,
        col: Union[str, Function, sqlalchemy.ColumnElement],
        side: Union[str, None],
    ):
        try:
            if isinstance(col, Function):
                return ds.c(col.get_column())
            return ds.c(col) if isinstance(col, (str, C)) else col
        except ValueError:
            if side:
                errors.append(f"{_get_merge_error_str(col)} in {side}")

    ops = [
        _resolve(self, left, "left")
        == _resolve(right_ds, right, "right" if right_on else None)
        for left, right in zip(on, right_on or on)
    ]

    if errors:
        raise DatasetMergeError(
            on, right_on, f"Could not resolve {', '.join(errors)}"
        )

    query = self._query.join(
        right_ds._query, sqlalchemy.and_(*ops), inner, full, rname + "{name}"
    )
    query.feature_schema = None
    ds = self._evolve(query=query)

    signals_schema = self.signals_schema.clone_without_sys_signals()
    right_signals_schema = right_ds.signals_schema.clone_without_sys_signals()
    ds.signals_schema = SignalSchema({"sys": Sys}) | signals_schema.merge(
        right_signals_schema, rname
    )

    return ds

min 

min(fr: DataType)

Compute the minimum of a column.

Source code in datachain/lib/dc/datachain.py 
1948
1949
1950
def min(self, fr: DataType):  # type: ignore[override]
    """Compute the minimum of a column."""
    return self._extend_to_data_model("min", fr)

mutate 

mutate(**kwargs) -> Self

Create new signals based on existing signals.

This method cannot modify existing columns. If you need to modify an existing column, use a different name for the new column and then use select() to choose which columns to keep.

This method is vectorized and more efficient compared to map(), and it does not extract or download any data from the internal database. However, it can only utilize predefined built-in functions and their combinations.

The supported functions

Numerical: +, -, *, /, rand(), avg(), count(), func(), greatest(), least(), max(), min(), sum() String: length(), split(), replace(), regexp_replace() Filename: name(), parent(), file_stem(), file_ext() Array: length(), sip_hash_64(), euclidean_distance(), cosine_distance() Window: row_number(), rank(), dense_rank(), first()

Example: 

 dc.mutate(
    area=Column("image.height") * Column("image.width"),
    extension=file_ext(Column("file.name")),
    dist=cosine_distance(embedding_text, embedding_image)
)

Window function example: 

window = func.window(partition_by="file.parent", order_by="file.size")
dc.mutate(
    row_number=func.row_number().over(window),
)

This method can be also used to rename signals. If the Column("name") provided as value for the new signal - the old column will be dropped. Otherwise a new column is created.

Example: 

 dc.mutate(
    newkey=Column("oldkey")
)

Source code in datachain/lib/dc/datachain.py 
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
def mutate(self, **kwargs) -> "Self":
    """Create new signals based on existing signals.

    This method cannot modify existing columns. If you need to modify an
    existing column, use a different name for the new column and then use
    `select()` to choose which columns to keep.

    This method is vectorized and more efficient compared to map(), and it does not
    extract or download any data from the internal database. However, it can only
    utilize predefined built-in functions and their combinations.

    The supported functions:
       Numerical:   +, -, *, /, rand(), avg(), count(), func(),
                    greatest(), least(), max(), min(), sum()
       String:      length(), split(), replace(), regexp_replace()
       Filename:    name(), parent(), file_stem(), file_ext()
       Array:       length(), sip_hash_64(), euclidean_distance(),
                    cosine_distance()
       Window:      row_number(), rank(), dense_rank(), first()

    Example:
    ```py
     dc.mutate(
        area=Column("image.height") * Column("image.width"),
        extension=file_ext(Column("file.name")),
        dist=cosine_distance(embedding_text, embedding_image)
    )
    ```

    Window function example:
    ```py
    window = func.window(partition_by="file.parent", order_by="file.size")
    dc.mutate(
        row_number=func.row_number().over(window),
    )
    ```

    This method can be also used to rename signals. If the Column("name") provided
    as value for the new signal - the old column will be dropped. Otherwise a new
    column is created.

    Example:
    ```py
     dc.mutate(
        newkey=Column("oldkey")
    )
    ```
    """
    from sqlalchemy.sql.sqltypes import NullType

    primitives = (bool, str, int, float)

    for col_name, expr in kwargs.items():
        if not isinstance(expr, (*primitives, Column, Func)) and isinstance(
            expr.type, NullType
        ):
            raise DataChainColumnError(
                col_name, f"Cannot infer type with expression {expr}"
            )

    mutated = {}
    schema = self.signals_schema
    for name, value in kwargs.items():
        if isinstance(value, Column):
            # renaming existing column
            for signal in schema.db_signals(name=value.name, as_columns=True):
                mutated[signal.name.replace(value.name, name, 1)] = signal  # type: ignore[union-attr]
        elif isinstance(value, Func):
            # adding new signal
            mutated[name] = value.get_column(schema)
        elif isinstance(value, primitives):
            # adding simple python constant primitives like str, int, float, bool
            val = literal(value)
            val.type = python_to_sql(type(value))()
            mutated[name] = val  # type: ignore[assignment]
        else:
            # adding new signal
            mutated[name] = value

    return self._evolve(
        query=self._query.mutate(**mutated), signal_schema=schema.mutate(kwargs)
    )

offset 

offset(offset: int) -> Self

Return the results starting with the offset row.

If the chain is unordered, which rows are skipped in undefined. If the chain has less than offset rows, the result is an empty chain.

Parameters:

  • offset (int) –

    Number of rows to skip.

Source code in datachain/lib/dc/datachain.py 
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
def offset(self, offset: int) -> "Self":
    """Return the results starting with the offset row.

    If the chain is unordered, which rows are skipped in undefined.
    If the chain has less than `offset` rows, the result is an empty chain.

    Parameters:
        offset (int): Number of rows to skip.
    """
    return self._evolve(query=self._query.offset(offset))

order_by 

order_by(*args, descending: bool = False) -> Self

Orders by specified set of columns.

Parameters:

  • descending (bool, default: False ) –

    Whether to sort in descending order or not.

Example 
dc.order_by("similarity_score", descending=True).limit(10)
Note

Order is not guaranteed when steps are added after an order_by statement. I.e. when using from_dataset an order_by statement should be used if the order of the records in the chain is important. Using order_by directly before limit, collect and collect_flatten will give expected results. See https://github.com/iterative/datachain/issues/477 for further details.

Source code in datachain/lib/dc/datachain.py 
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
@resolve_columns
def order_by(self, *args, descending: bool = False) -> "Self":
    """Orders by specified set of columns.

    Parameters:
        descending (bool): Whether to sort in descending order or not.

    Example:
        ```py
        dc.order_by("similarity_score", descending=True).limit(10)
        ```

    Note:
        Order is not guaranteed when steps are added after an `order_by` statement.
        I.e. when using `from_dataset` an `order_by` statement should be used if
        the order of the records in the chain is important.
        Using `order_by` directly before `limit`, `collect` and `collect_flatten`
        will give expected results.
        See https://github.com/iterative/datachain/issues/477 for further details.
    """
    if descending:
        args = tuple(sqlalchemy.desc(a) for a in args)

    return self._evolve(query=self._query.order_by(*args))

parse_tabular 

parse_tabular(
    output: OutputType = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs
) -> Self

Generate chain from list of tabular files.

Parameters:

  • output –

    Dictionary or feature class defining column names and their corresponding types. List of column names is also accepted, in which case types will be inferred.

  • object_name –

    Generated object column name.

  • model_name –

    Generated model name.

  • source –

    Whether to include info about the source file.

  • nrows –

    Optional row limit.

  • kwargs –

    Parameters to pass to pyarrow.dataset.dataset.

Example

Reading a json lines file: 

dc = DataChain.from_storage("s3://mybucket/file.jsonl")
dc = dc.parse_tabular(format="json")

Reading a filtered list of files as a dataset: 

dc = DataChain.from_storage("s3://mybucket")
dc = dc.filter(C("file.name").glob("*.jsonl"))
dc = dc.parse_tabular(format="json")

Source code in datachain/lib/dc/datachain.py 
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
def parse_tabular(
    self,
    output: OutputType = None,
    object_name: str = "",
    model_name: str = "",
    source: bool = True,
    nrows: Optional[int] = None,
    **kwargs,
) -> "Self":
    """Generate chain from list of tabular files.

    Parameters:
        output : Dictionary or feature class defining column names and their
            corresponding types. List of column names is also accepted, in which
            case types will be inferred.
        object_name : Generated object column name.
        model_name : Generated model name.
        source : Whether to include info about the source file.
        nrows : Optional row limit.
        kwargs : Parameters to pass to pyarrow.dataset.dataset.

    Example:
        Reading a json lines file:
        ```py
        dc = DataChain.from_storage("s3://mybucket/file.jsonl")
        dc = dc.parse_tabular(format="json")
        ```

        Reading a filtered list of files as a dataset:
        ```py
        dc = DataChain.from_storage("s3://mybucket")
        dc = dc.filter(C("file.name").glob("*.jsonl"))
        dc = dc.parse_tabular(format="json")
        ```
    """
    from pyarrow.dataset import CsvFileFormat, JsonFileFormat

    from datachain.lib.arrow import ArrowGenerator, infer_schema, schema_to_output

    if nrows:
        format = kwargs.get("format")
        if format not in ["csv", "json"] and not isinstance(
            format, (CsvFileFormat, JsonFileFormat)
        ):
            raise DatasetPrepareError(
                self.name,
                "error in `parse_tabular` - "
                "`nrows` only supported for csv and json formats.",
            )

    if "file" not in self.schema or not self.count():
        raise DatasetPrepareError(self.name, "no files to parse.")

    schema = None
    col_names = output if isinstance(output, Sequence) else None
    if col_names or not output:
        try:
            schema = infer_schema(self, **kwargs)
            output, _ = schema_to_output(schema, col_names)
        except ValueError as e:
            raise DatasetPrepareError(self.name, e) from e

    if isinstance(output, dict):
        model_name = model_name or object_name or ""
        model = dict_to_data_model(model_name, output)
        output = model
    else:
        model = output  # type: ignore[assignment]

    if object_name:
        output = {object_name: model}  # type: ignore[dict-item]
    elif isinstance(output, type(BaseModel)):
        output = {
            name: info.annotation  # type: ignore[misc]
            for name, info in output.model_fields.items()
        }

    if source:
        output = {"source": ArrowRow} | output  # type: ignore[assignment,operator]

    # disable prefetch if nrows is set
    settings = {"prefetch": 0} if nrows else {}
    return self.settings(**settings).gen(  # type: ignore[arg-type]
        ArrowGenerator(schema, model, source, nrows, **kwargs), output=output
    )

print_schema 

print_schema(file: Optional[IO] = None) -> None

Print schema of the chain.

Source code in datachain/lib/dc/datachain.py 
232
233
234
def print_schema(self, file: Optional[IO] = None) -> None:
    """Print schema of the chain."""
    self._effective_signals_schema.print_tree(file=file)

reset_settings 

reset_settings(settings: Optional[Settings] = None) -> Self

Reset all settings to default values.

Source code in datachain/lib/dc/datachain.py 
299
300
301
302
def reset_settings(self, settings: Optional[Settings] = None) -> "Self":
    """Reset all settings to default values."""
    self._settings = settings if settings else Settings()
    return self

sample 

sample(n) -> Self

Return a random sample from the chain.

Parameters:

  • n (int) –

    Number of samples to draw.

NOTE: Samples are not deterministic, and streamed/paginated queries or multiple workers will draw samples with replacement.

Source code in datachain/lib/dc/datachain.py 
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
def sample(self, n) -> "Self":
    """Return a random sample from the chain.

    Parameters:
        n (int): Number of samples to draw.

    NOTE: Samples are not deterministic, and streamed/paginated queries or
    multiple workers will draw samples with replacement.
    """
    return self._evolve(query=self._query.sample(n))

save 

save(
    name: Optional[str] = None,
    version: Optional[int] = None,
    description: Optional[str] = None,
    labels: Optional[list[str]] = None,
    **kwargs
) -> Self

Save to a Dataset. It returns the chain itself.

Parameters:

  • name –

    dataset name. Empty name saves to a temporary dataset that will be removed after process ends. Temp dataset are useful for optimization.

  • version –

    version of a dataset. Default - the last version that exist.

  • description –

    description of a dataset.

  • labels –

    labels of a dataset.

Source code in datachain/lib/dc/datachain.py 
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
def save(  # type: ignore[override]
    self,
    name: Optional[str] = None,
    version: Optional[int] = None,
    description: Optional[str] = None,
    labels: Optional[list[str]] = None,
    **kwargs,
) -> "Self":
    """Save to a Dataset. It returns the chain itself.

    Parameters:
        name : dataset name. Empty name saves to a temporary dataset that will be
            removed after process ends. Temp dataset are useful for optimization.
        version : version of a dataset. Default - the last version that exist.
        description : description of a dataset.
        labels : labels of a dataset.
    """
    schema = self.signals_schema.clone_without_sys_signals().serialize()
    return self._evolve(
        query=self._query.save(
            name=name,
            version=version,
            description=description,
            labels=labels,
            feature_schema=schema,
            **kwargs,
        )
    )

select 

select(*args: str, _sys: bool = True) -> Self

Select only a specified set of signals.

Source code in datachain/lib/dc/datachain.py 
757
758
759
760
761
762
763
764
765
def select(self, *args: str, _sys: bool = True) -> "Self":
    """Select only a specified set of signals."""
    new_schema = self.signals_schema.resolve(*args)
    if self._sys and _sys:
        new_schema = SignalSchema({"sys": Sys}) | new_schema
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

select_except 

select_except(*args: str) -> Self

Select all the signals expect the specified signals.

Source code in datachain/lib/dc/datachain.py 
767
768
769
770
771
772
773
def select_except(self, *args: str) -> "Self":
    """Select all the signals expect the specified signals."""
    new_schema = self.signals_schema.select_except_signals(*args)
    columns = new_schema.db_signals()
    return self._evolve(
        query=self._query.select(*columns), signal_schema=new_schema
    )

settings 

settings(
    cache=None,
    parallel=None,
    workers=None,
    min_task_size=None,
    prefetch: Optional[int] = None,
    sys: Optional[bool] = None,
) -> Self

Change settings for chain.

This function changes specified settings without changing not specified ones. It returns chain, so, it can be chained later with next operation.

Parameters:

  • cache –

    data caching (default=False)

  • parallel –

    number of thread for processors. True is a special value to enable all available CPUs (default=1)

  • workers –

    number of distributed workers. Only for Studio mode. (default=1)

  • min_task_size –

    minimum number of tasks (default=1)

  • prefetch (Optional[int], default: None ) –

    number of workers to use for downloading files in advance. This is enabled by default and uses 2 workers. To disable prefetching, set it to 0.

Example 
chain = (
    chain
    .settings(cache=True, parallel=8)
    .map(laion=process_webdataset(spec=WDSLaion), params="file")
)
Source code in datachain/lib/dc/datachain.py 
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
def settings(
    self,
    cache=None,
    parallel=None,
    workers=None,
    min_task_size=None,
    prefetch: Optional[int] = None,
    sys: Optional[bool] = None,
) -> "Self":
    """Change settings for chain.

    This function changes specified settings without changing not specified ones.
    It returns chain, so, it can be chained later with next operation.

    Parameters:
        cache : data caching (default=False)
        parallel : number of thread for processors. True is a special value to
            enable all available CPUs (default=1)
        workers : number of distributed workers. Only for Studio mode. (default=1)
        min_task_size : minimum number of tasks (default=1)
        prefetch: number of workers to use for downloading files in advance.
                  This is enabled by default and uses 2 workers.
                  To disable prefetching, set it to 0.

    Example:
        ```py
        chain = (
            chain
            .settings(cache=True, parallel=8)
            .map(laion=process_webdataset(spec=WDSLaion), params="file")
        )
        ```
    """
    if sys is None:
        sys = self._sys
    settings = copy.copy(self._settings)
    settings.add(Settings(cache, parallel, workers, min_task_size, prefetch))
    return self._evolve(settings=settings, _sys=sys)

setup 

setup(**kwargs) -> Self

Setup variables to pass to UDF functions.

Use before running map/gen/agg/batch_map to save an object and pass it as an argument to the UDF.

Example 
import anthropic
from anthropic.types import Message

(
    DataChain.from_storage(DATA, type="text")
    .settings(parallel=4, cache=True)
    .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
    .map(
        claude=lambda client, file: client.messages.create(
            model=MODEL,
            system=PROMPT,
            messages=[{"role": "user", "content": file.get_value()}],
        ),
        output=Message,
    )
)
Source code in datachain/lib/dc/datachain.py 
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
def setup(self, **kwargs) -> "Self":
    """Setup variables to pass to UDF functions.

    Use before running map/gen/agg/batch_map to save an object and pass it as an
    argument to the UDF.

    Example:
        ```py
        import anthropic
        from anthropic.types import Message

        (
            DataChain.from_storage(DATA, type="text")
            .settings(parallel=4, cache=True)
            .setup(client=lambda: anthropic.Anthropic(api_key=API_KEY))
            .map(
                claude=lambda client, file: client.messages.create(
                    model=MODEL,
                    system=PROMPT,
                    messages=[{"role": "user", "content": file.get_value()}],
                ),
                output=Message,
            )
        )
        ```
    """
    intersection = set(self._setup.keys()) & set(kwargs.keys())
    if intersection:
        keys = ", ".join(intersection)
        raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")

    self._setup = self._setup | kwargs
    return self

show 

show(
    limit: int = 20,
    flatten=False,
    transpose=False,
    truncate=True,
    include_hidden=False,
) -> None

Show a preview of the chain results.

Parameters:

  • limit –

    How many rows to show.

  • flatten –

    Whether to use a multiindex or flatten column names.

  • transpose –

    Whether to transpose rows and columns.

  • truncate –

    Whether or not to truncate the contents of columns.

  • include_hidden –

    Whether to include hidden columns.

Source code in datachain/lib/dc/datachain.py 
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
def show(
    self,
    limit: int = 20,
    flatten=False,
    transpose=False,
    truncate=True,
    include_hidden=False,
) -> None:
    """Show a preview of the chain results.

    Parameters:
        limit : How many rows to show.
        flatten : Whether to use a multiindex or flatten column names.
        transpose : Whether to transpose rows and columns.
        truncate : Whether or not to truncate the contents of columns.
        include_hidden : Whether to include hidden columns.
    """
    import pandas as pd

    dc = self.limit(limit) if limit > 0 else self  # type: ignore[misc]
    df = dc.to_pandas(flatten, include_hidden=include_hidden)

    if df.empty:
        print("Empty result")
        print(f"Columns: {list(df.columns)}")
        return

    if transpose:
        df = df.T

    options: list = [
        "display.max_columns",
        None,
        "display.multi_sparse",
        False,
    ]

    try:
        if columns := os.get_terminal_size().columns:
            options.extend(["display.width", columns])
    except OSError:
        pass

    if not truncate:
        options.extend(["display.max_colwidth", None])

    with pd.option_context(*options):
        if inside_notebook():
            from IPython.display import display

            display(df)
        else:
            print(df)

    if len(df) == limit:
        print(f"\n[Limited by {len(df)} rows]")

shuffle 

shuffle() -> Self

Shuffle the rows of the chain deterministically.

Source code in datachain/lib/dc/datachain.py 
2049
2050
2051
def shuffle(self) -> "Self":
    """Shuffle the rows of the chain deterministically."""
    return self.order_by("sys.rand")

subtract 

subtract(
    other: DataChain,
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> Self

Remove rows that appear in another chain.

Parameters:

  • other (DataChain) –

    chain whose rows will be removed from self

  • on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in self. If unspecified, defaults to all common columns between self and other.

  • right_on (Optional[Union[str, Sequence[str]]], default: None ) –

    columns to consider for determining row equality in other. If unspecified, defaults to the same values as on.

Source code in datachain/lib/dc/datachain.py 
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
def subtract(  # type: ignore[override]
    self,
    other: "DataChain",
    on: Optional[Union[str, Sequence[str]]] = None,
    right_on: Optional[Union[str, Sequence[str]]] = None,
) -> "Self":
    """Remove rows that appear in another chain.

    Parameters:
        other: chain whose rows will be removed from `self`
        on: columns to consider for determining row equality in `self`.
            If unspecified, defaults to all common columns
            between `self` and `other`.
        right_on: columns to consider for determining row equality in `other`.
            If unspecified, defaults to the same values as `on`.
    """
    if isinstance(on, str):
        if not on:
            raise DataChainParamsError("'on' cannot be an empty string")
        on = [on]
    elif isinstance(on, Sequence):
        if not on or any(not col for col in on):
            raise DataChainParamsError("'on' cannot contain empty strings")

    if isinstance(right_on, str):
        if not right_on:
            raise DataChainParamsError("'right_on' cannot be an empty string")
        right_on = [right_on]
    elif isinstance(right_on, Sequence):
        if not right_on or any(not col for col in right_on):
            raise DataChainParamsError("'right_on' cannot contain empty strings")

    if on is None and right_on is None:
        other_columns = set(other._effective_signals_schema.db_signals())
        signals = [
            c
            for c in self._effective_signals_schema.db_signals()
            if c in other_columns
        ]
        if not signals:
            raise DataChainParamsError("subtract(): no common columns")
    elif on is not None and right_on is None:
        right_on = on
        signals = list(self.signals_schema.resolve(*on).db_signals())
    elif on is None and right_on is not None:
        raise DataChainParamsError(
            "'on' must be specified when 'right_on' is provided"
        )
    else:
        if not isinstance(on, Sequence) or not isinstance(right_on, Sequence):
            raise TypeError(
                "'on' and 'right_on' must be 'str' or 'Sequence' object"
            )
        if len(on) != len(right_on):
            raise DataChainParamsError(
                "'on' and 'right_on' must have the same length"
            )
        signals = list(
            zip(
                self.signals_schema.resolve(*on).db_signals(),
                other.signals_schema.resolve(*right_on).db_signals(),
            )  # type: ignore[arg-type]
        )
    return self._evolve(query=self._query.subtract(other._query, signals))  # type: ignore[arg-type]

sum 

sum(fr: DataType)

Compute the sum of a column.

Source code in datachain/lib/dc/datachain.py 
1940
1941
1942
def sum(self, fr: DataType):  # type: ignore[override]
    """Compute the sum of a column."""
    return self._extend_to_data_model("sum", fr)

to_columnar_data_with_names 

to_columnar_data_with_names(
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
) -> tuple[list[str], Iterator[list[list[Any]]]]

Returns column names and the results as an iterator that provides chunks, with each chunk containing a list of columns, where each column contains a list of the row values for that column in that chunk. Useful for columnar data formats, such as parquet or other OLAP databases.

Source code in datachain/lib/dc/datachain.py 
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
def to_columnar_data_with_names(
    self, chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE
) -> tuple[list[str], Iterator[list[list[Any]]]]:
    """Returns column names and the results as an iterator that provides chunks,
    with each chunk containing a list of columns, where each column contains a
    list of the row values for that column in that chunk. Useful for columnar data
    formats, such as parquet or other OLAP databases.
    """
    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self.collect_flatten()

    def column_chunks() -> Iterator[list[list[Any]]]:
        for chunk_iter in batched_it(results_iter, chunk_size):
            columns: list[list[Any]] = [[] for _ in column_names]
            for row in chunk_iter:
                for i, col in enumerate(columns):
                    col.append(row[i])
            yield columns

    return column_names, column_chunks()

to_csv 

to_csv(
    path: Union[str, PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to a csv (comma-separated values) file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • delimiter –

    Delimiter to use for the resulting file.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
def to_csv(
    self,
    path: Union[str, os.PathLike[str]],
    delimiter: str = ",",
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to a csv (comma-separated values) file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        delimiter : Delimiter to use for the resulting file.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import csv

    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    column_names = [".".join(filter(None, header)) for header in headers]

    results_iter = self.collect_flatten()

    with opener(path, "w", newline="") as f:
        writer = csv.writer(f, delimiter=delimiter, **kwargs)
        writer.writerow(column_names)

        for row in results_iter:
            writer.writerow(row)

to_json 

to_json(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None

Save chain to a JSON file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

  • include_outer_list –

    Sets whether to include an outer list for all rows. Setting this to True makes the file valid JSON, while False instead writes in the JSON lines format.

Source code in datachain/lib/dc/datachain.py 
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
def to_json(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
    include_outer_list: bool = True,
) -> None:
    """Save chain to a JSON file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
        include_outer_list : Sets whether to include an outer list for all rows.
            Setting this to True makes the file valid JSON, while False instead
            writes in the JSON lines format.
    """
    opener = open

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        fsspec_fs = client.create_fs(**fs_kwargs)

        opener = fsspec_fs.open

    headers, _ = self._effective_signals_schema.get_headers_with_length()
    headers = [list(filter(None, header)) for header in headers]

    is_first = True

    with opener(path, "wb") as f:
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"[\n")
        for row in self.collect_flatten():
            if not is_first:
                if include_outer_list:
                    # This makes the file JSON instead of JSON lines.
                    f.write(b",\n")
                else:
                    f.write(b"\n")
            else:
                is_first = False
            f.write(orjson.dumps(row_to_nested_dict(headers, row)))
        if include_outer_list:
            # This makes the file JSON instead of JSON lines.
            f.write(b"\n]\n")

to_jsonl 

to_jsonl(
    path: Union[str, PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None

Save chain to a JSON lines file.

Parameters:

  • path –

    Path to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
def to_jsonl(
    self,
    path: Union[str, os.PathLike[str]],
    fs_kwargs: Optional[dict[str, Any]] = None,
) -> None:
    """Save chain to a JSON lines file.

    Parameters:
        path : Path to save the file. This supports local paths as well as
            remote paths, such as s3:// or hf:// with fsspec.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    self.to_json(path, fs_kwargs, include_outer_list=False)

to_pandas 

to_pandas(flatten=False, include_hidden=True) -> DataFrame

Return a pandas DataFrame from the chain.

Parameters:

  • flatten –

    Whether to use a multiindex or flatten column names.

  • include_hidden –

    Whether to include hidden columns.

Source code in datachain/lib/dc/datachain.py 
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
def to_pandas(self, flatten=False, include_hidden=True) -> "pd.DataFrame":
    """Return a pandas DataFrame from the chain.

    Parameters:
        flatten : Whether to use a multiindex or flatten column names.
        include_hidden : Whether to include hidden columns.
    """
    import pandas as pd

    headers, max_length = self._effective_signals_schema.get_headers_with_length(
        include_hidden=include_hidden
    )
    if flatten or max_length < 2:
        columns = [".".join(filter(None, header)) for header in headers]
    else:
        columns = pd.MultiIndex.from_tuples(map(tuple, headers))

    results = self.results(include_hidden=include_hidden)
    return pd.DataFrame.from_records(results, columns=columns)

to_parquet 

to_parquet(
    path: Union[str, PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs
) -> None

Save chain to parquet file with SignalSchema metadata.

Parameters:

  • path –

    Path or a file-like binary object to save the file. This supports local paths as well as remote paths, such as s3:// or hf:// with fsspec.

  • partition_cols –

    Column names by which to partition the dataset.

  • chunk_size –

    The chunk size of results to read and convert to columnar data, to avoid running out of memory.

  • fs_kwargs –

    Optional kwargs to pass to the fsspec filesystem, used only for write, for fsspec-type URLs, such as s3:// or hf:// when provided as the destination path.

Source code in datachain/lib/dc/datachain.py 
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
def to_parquet(
    self,
    path: Union[str, os.PathLike[str], BinaryIO],
    partition_cols: Optional[Sequence[str]] = None,
    chunk_size: int = DEFAULT_PARQUET_CHUNK_SIZE,
    fs_kwargs: Optional[dict[str, Any]] = None,
    **kwargs,
) -> None:
    """Save chain to parquet file with SignalSchema metadata.

    Parameters:
        path : Path or a file-like binary object to save the file. This supports
            local paths as well as remote paths, such as s3:// or hf:// with fsspec.
        partition_cols : Column names by which to partition the dataset.
        chunk_size : The chunk size of results to read and convert to columnar
            data, to avoid running out of memory.
        fs_kwargs : Optional kwargs to pass to the fsspec filesystem, used only for
            write, for fsspec-type URLs, such as s3:// or hf:// when
            provided as the destination path.
    """
    import pyarrow as pa
    import pyarrow.parquet as pq

    from datachain.lib.arrow import DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY

    fsspec_fs = None

    if isinstance(path, str) and "://" in path:
        from datachain.client.fsspec import Client

        fs_kwargs = {
            **self._query.catalog.client_config,
            **(fs_kwargs or {}),
        }

        client = Client.get_implementation(path)

        if path.startswith("file://"):
            # pyarrow does not handle file:// uris, and needs a direct path instead.
            from urllib.parse import urlparse

            path = urlparse(path).path
            if sys.platform == "win32":
                path = os.path.normpath(path.lstrip("/"))

        fsspec_fs = client.create_fs(**fs_kwargs)

    _partition_cols = list(partition_cols) if partition_cols else None
    signal_schema_metadata = orjson.dumps(
        self._effective_signals_schema.serialize()
    )

    column_names, column_chunks = self.to_columnar_data_with_names(chunk_size)

    parquet_schema = None
    parquet_writer = None
    first_chunk = True

    for chunk in column_chunks:
        # pyarrow infers the best parquet schema from the python types of
        # the input data.
        table = pa.Table.from_pydict(
            dict(zip(column_names, chunk)),
            schema=parquet_schema,
        )

        # Preserve any existing metadata, and add the DataChain SignalSchema.
        existing_metadata = table.schema.metadata or {}
        merged_metadata = {
            **existing_metadata,
            DATACHAIN_SIGNAL_SCHEMA_PARQUET_KEY: signal_schema_metadata,
        }
        table = table.replace_schema_metadata(merged_metadata)
        parquet_schema = table.schema

        if _partition_cols:
            # Write to a partitioned parquet dataset.
            pq.write_to_dataset(
                table,
                root_path=path,
                partition_cols=_partition_cols,
                filesystem=fsspec_fs,
                **kwargs,
            )
        else:
            if first_chunk:
                # Write to a single parquet file.
                parquet_writer = pq.ParquetWriter(
                    path, parquet_schema, filesystem=fsspec_fs, **kwargs
                )
                first_chunk = False

            assert parquet_writer
            parquet_writer.write_table(table)

    if parquet_writer:
        parquet_writer.close()

to_pytorch 

to_pytorch(
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
)

Convert to pytorch dataset format.

Parameters:

  • transform (Transform, default: None ) –

    Torchvision transforms to apply to the dataset.

  • tokenizer (Callable, default: None ) –

    Tokenizer to use to tokenize text values.

  • tokenizer_kwargs (dict, default: None ) –

    Additional kwargs to pass when calling tokenizer.

  • num_samples (int, default: 0 ) –

    Number of random samples to draw for each epoch. This argument is ignored if num_samples=0 (the default).

  • remove_prefetched (bool, default: False ) –

    Whether to remove prefetched files after reading.

Example 
from torch.utils.data import DataLoader
loader = DataLoader(
    chain.select("file", "label").to_pytorch(),
    batch_size=16
)
Source code in datachain/lib/dc/datachain.py 
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
def to_pytorch(
    self,
    transform=None,
    tokenizer=None,
    tokenizer_kwargs=None,
    num_samples=0,
    remove_prefetched: bool = False,
):
    """Convert to pytorch dataset format.

    Args:
        transform (Transform): Torchvision transforms to apply to the dataset.
        tokenizer (Callable): Tokenizer to use to tokenize text values.
        tokenizer_kwargs (dict): Additional kwargs to pass when calling tokenizer.
        num_samples (int): Number of random samples to draw for each epoch.
            This argument is ignored if `num_samples=0` (the default).
        remove_prefetched (bool): Whether to remove prefetched files after reading.

    Example:
        ```py
        from torch.utils.data import DataLoader
        loader = DataLoader(
            chain.select("file", "label").to_pytorch(),
            batch_size=16
        )
        ```
    """
    from datachain.torch import PytorchDataset

    if self._query.attached:
        chain = self
    else:
        chain = self.save()
    assert chain.name is not None  # for mypy
    return PytorchDataset(
        chain.name,
        chain.version,
        catalog=self.session.catalog,
        transform=transform,
        tokenizer=tokenizer,
        tokenizer_kwargs=tokenizer_kwargs,
        num_samples=num_samples,
        dc_settings=chain._settings,
        remove_prefetched=remove_prefetched,
    )

to_records 

to_records() -> list[dict[str, Any]]

Convert every row to a dictionary.

Source code in datachain/lib/dc/datachain.py 
1026
1027
1028
1029
1030
1031
1032
def to_records(self) -> list[dict[str, Any]]:
    """Convert every row to a dictionary."""

    def to_dict(cols: list[str], row: tuple[Any, ...]) -> dict[str, Any]:
        return dict(zip(cols, row))

    return self.results(row_factory=to_dict)

to_storage 

to_storage(
    output: Union[str, PathLike[str]],
    signal: str = "file",
    placement: ExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: bool = False,
    client_config: Optional[dict] = None,
) -> None

Export files from a specified signal to a directory. Files can be exported to a local or cloud directory.

Parameters:

  • output (Union[str, PathLike[str]]) –

    Path to the target directory for exporting files.

  • signal (str, default: 'file' ) –

    Name of the signal to export files from.

  • placement (ExportPlacement, default: 'fullpath' ) –

    The method to use for naming exported files. The possible values are: "filename", "etag", "fullpath", and "checksum".

  • link_type (Literal['copy', 'symlink'], default: 'copy' ) –

    Method to use for exporting files. Falls back to 'copy' if symlinking fails.

  • num_threads –

    number of threads to use for exporting files. By default it uses 5 threads.

  • anon (bool, default: False ) –

    If true, we will treat cloud bucket as public one

  • client_config (Optional[dict], default: None ) –

    Optional configuration for the destination storage client

Example

Cross cloud transfer 

ds = DataChain.from_storage("s3://mybucket")
ds.to_storage("gs://mybucket", placement="filename")

Source code in datachain/lib/dc/datachain.py 
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
def to_storage(
    self,
    output: Union[str, os.PathLike[str]],
    signal: str = "file",
    placement: FileExportPlacement = "fullpath",
    link_type: Literal["copy", "symlink"] = "copy",
    num_threads: Optional[int] = EXPORT_FILES_MAX_THREADS,
    anon: bool = False,
    client_config: Optional[dict] = None,
) -> None:
    """Export files from a specified signal to a directory. Files can be
    exported to a local or cloud directory.

    Args:
        output: Path to the target directory for exporting files.
        signal: Name of the signal to export files from.
        placement: The method to use for naming exported files.
            The possible values are: "filename", "etag", "fullpath", and "checksum".
        link_type: Method to use for exporting files.
            Falls back to `'copy'` if symlinking fails.
        num_threads : number of threads to use for exporting files.
            By default it uses 5 threads.
        anon: If true, we will treat cloud bucket as public one
        client_config: Optional configuration for the destination storage client

    Example:
        Cross cloud transfer
        ```py
        ds = DataChain.from_storage("s3://mybucket")
        ds.to_storage("gs://mybucket", placement="filename")
        ```
    """
    if placement == "filename" and (
        self._query.distinct(pathfunc.name(C(f"{signal}__path"))).count()
        != self._query.count()
    ):
        raise ValueError("Files with the same name found")

    if anon:
        client_config = (client_config or {}) | {"anon": True}

    progress_bar = tqdm(
        desc=f"Exporting files to {output}: ",
        unit=" files",
        unit_scale=True,
        unit_divisor=10,
        total=self.count(),
        leave=False,
    )
    file_exporter = FileExporter(
        output,
        placement,
        self._settings.cache if self._settings else False,
        link_type,
        max_threads=num_threads or 1,
        client_config=client_config,
    )
    file_exporter.run(self.collect(signal), progress_bar)

union 

union(other: Self) -> Self

Return the set union of the two datasets.

Parameters:

  • other (Self) –

    chain whose rows will be added to self.

Source code in datachain/lib/dc/datachain.py 
1241
1242
1243
1244
1245
1246
1247
def union(self, other: "Self") -> "Self":
    """Return the set union of the two datasets.

    Parameters:
        other: chain whose rows will be added to `self`.
    """
    return self._evolve(query=self._query.union(other._query))

DataChainError 

DataChainError(message)

Bases: Exception

Source code in datachain/lib/utils.py 
21
22
def __init__(self, message):
    super().__init__(message)

Session 

Session(
    name="",
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
)

Session is a context that keeps track of temporary DataChain datasets for a proper cleanup. By default, a global session is created.

Temporary or ephemeral datasets are the ones created without specified name. They are useful for optimization purposes and should be automatically removed.

Temp dataset has specific name format

"session_"

The suffix is optional. Both s are auto-generated.

Temp dataset examples

session_myname_624b41_48e8b4 session_4b962d_2a5dff

Parameters:

name (str): The name of the session. Only latters and numbers are supported. It can be empty. catalog (Catalog): Catalog object.

Source code in datachain/query/session.py 
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def __init__(
    self,
    name="",
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
):
    if re.match(r"^[0-9a-zA-Z]*$", name) is None:
        raise ValueError(
            f"Session name can contain only letters or numbers - '{name}' given."
        )

    if not name:
        name = self.GLOBAL_SESSION_NAME

    session_uuid = uuid4().hex[: self.SESSION_UUID_LEN]
    self.name = f"{name}_{session_uuid}"
    self.is_new_catalog = not catalog
    self.catalog = catalog or get_catalog(
        client_config=client_config, in_memory=in_memory
    )
    self.dataset_versions: list[tuple[DatasetRecord, int, bool]] = []

get classmethod 

get(
    session: Optional[Session] = None,
    catalog: Optional[Catalog] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> Session

Creates a Session() object from a catalog.

Parameters:

  • session (Session, default: None ) –

    Optional Session(). If not provided a new session will be created. It's needed mostly for simple API purposes.

  • catalog (Catalog, default: None ) –

    Optional catalog. By default, a new catalog is created.

Source code in datachain/query/session.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
@classmethod
def get(
    cls,
    session: Optional["Session"] = None,
    catalog: Optional["Catalog"] = None,
    client_config: Optional[dict] = None,
    in_memory: bool = False,
) -> "Session":
    """Creates a Session() object from a catalog.

    Parameters:
        session (Session): Optional Session(). If not provided a new session will
                be created. It's needed mostly for simple API purposes.
        catalog (Catalog): Optional catalog. By default, a new catalog is created.
    """
    if session:
        return session

    # Access the active (most recent) context from the stack
    if cls.SESSION_CONTEXTS:
        session = cls.SESSION_CONTEXTS[-1]

    elif cls.GLOBAL_SESSION_CTX is None:
        cls.GLOBAL_SESSION_CTX = Session(
            cls.GLOBAL_SESSION_NAME,
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session = cls.GLOBAL_SESSION_CTX

        atexit.register(cls._global_cleanup)
        cls.ORIGINAL_EXCEPT_HOOK = sys.excepthook
        sys.excepthook = cls.except_hook
    else:
        session = cls.GLOBAL_SESSION_CTX

    if client_config and session.catalog.client_config != client_config:
        session = Session(
            "session" + uuid4().hex[:4],
            catalog,
            client_config=client_config,
            in_memory=in_memory,
        )
        session.__enter__()

    return session

Sys

Bases: DataModel

Model for internal DataChain signals id and rand.