Resources

ObjectStorage

Declaration

from dagster_toolbox.resources import ObjectStorage


class Analytics(ObjectStorage):
    bucket = "analytics"
    

@resource
def analytics_objects(init_context):
    return Analytics()


RESOURCE_DEFS = {
    "analytics_objects": analytics_objects,
}

Usage

from dagster import asset


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    partition_key = context.asset_partition_key_for_output()
    file_path = f"analytics/data/{partition_key}"
    
    file_resource = context.resource.analytics_objects.read(file_path)
    ...

csv_partitioned_io_manager

Declaration

from dagster_toolbox.resources import csv_partitioned_io_manager


RESOURCE_DEFS = {
    "csv_partitioned_io_manager": csv_partitioned_io_manager.configured(
        {
            "s3_bucket": "analytics",
        }
    ),
}

Usage

from dagster import asset

from analytics.resource_defs import RESOURCE_DEFS


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    ...

json_partitioned_io_manager

Declaration

from dagster_toolbox.resources import json_partitioned_io_manager


RESOURCE_DEFS = {
    "json_partitioned_io_manager": json_partitioned_io_manager.configured(
        {
            "s3_bucket": "analytics",
        }
    ),
}

Usage

from dagster import asset

from analytics.resource_defs import RESOURCE_DEFS


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    ...

postgres_partitioned_io_manager

Declaration

from dagster_toolbox.resources import postgres_partitioned_io_manager


RESOURCE_DEFS = {
    "postgres_partitioned_io_manager": postgres_partitioned_io_manager.configured(
        {
            "partition_expr": "customer_key,partition_key",
            "hostname": {
                "env": "POSTGRES_HOSTNAME",
            },
            "username": {
                "env": "POSTGRES_USER",
            },
            "password": {
                "env": "POSTGRES_PASSWORD",
            },
            "port": {
                "env": "POSTGRES_PORT",
            },
            "dbname": "analytics",
        }
    ),
}

Usage

from dagster import asset

from analytics.resource_defs import RESOURCE_DEFS


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    ...

postgres_io_manager

Declaration

from dagster_toolbox.resources import postgres_io_manager


RESOURCE_DEFS = {
    "postgres_io_manager": postgres_io_manager.configured(
        {
            "hostname": {
                "env": "POSTGRES_HOSTNAME",
            },
            "username": {
                "env": "POSTGRES_USER",
            },
            "password": {
                "env": "POSTGRES_PASSWORD",
            },
            "port": {
                "env": "POSTGRES_PORT",
            },
            "dbname": "analytics",
        }
    ),
}

Usage

from dagster import asset

from analytics.resource_defs import RESOURCE_DEFS


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    ...

vault

Declaration

from dagster_toolbox.resources import vault


RESOURCE_DEFS = {
    "vault": vault
}

Usage

from dagster import asset

from analytics.resource_defs import RESOURCE_DEFS


@asset(resource_defs=RESOURCE_DEFS,)
def export_files(context):
    secrets = context.resources.vault.get_data(SECRET_PATH)
    ...

results matching ""

    No results matching ""