sfujiwara / kfpc

Components for Kubeflow Pipelines and Vertex Pipelines.

KFPC: Kubeflow Pipelines Components

Docs

Components for Kubeflow Pipelines and Vertex Pipelines.

Installation

pip install git+https://github.com/sfujiwara/kfpc@${RELEASE_TAG}

Basic Usage

Components for BigQuery

import kfp
from kfp.v2 import compiler
from google.cloud import aiplatform
import kfpc


@kfp.dsl.pipeline(name="simple")
def pipeline_fn(project: str):
    query_task = kfpc.bigquery.Query(name="query").task(
        job_project=project,
        query="SELECT 1",
        destination_project=project,
        destination_dataset="sandbox",
        destination_table="tmp",
    )


project = "<Your GCP Project>"

compiler.Compiler().compile(pipeline_func=pipeline_fn, package_path="pipeline.json")
job = aiplatform.PipelineJob(
    project=project,
    display_name="simple",
    template_path="pipeline.json",
    parameter_values={"project": project},
    pipeline_root=f"gs://{project}-vertex/pipeline-root",
)
job.submit()