from data_validator import DataValidator

# Initialize validator with configuration
validator = DataValidator("config.yaml")

# Validate a table
summary = validator.validate_table(df, "customers")

# Generate report
report = validator.get_validation_report(summary)
print(f"Success rate: {summary.overall_success_rate:.2%}")

# Apply rules as filters to clean data
clean_df = validator.apply_filters(df, "customers")

from data_validator import DataValidator

# In your DLT pipeline
validator = DataValidator("dlt_config.yaml")

@dlt.table(
    name="validated_customers",
    comment="Customer data with quality validation"
)
def validated_customers():
    df = spark.table("raw_customers")
    
    # Validate data
    summary = validator.validate_with_dlt(df, "customers", dlt_expectations=True)
    
    # Apply filters for clean data
    return validator.apply_filters(df, "customers")

from data_validator import DataValidator

# Initialize validator with Databricks configuration
validator = DataValidator("databricks_config.yaml")

# Validate Unity Catalog table
df = spark.table("main.customer_data.customers")
summary = validator.validate_table(df, "customers")

# Get cluster information
cluster_info = validator.engine.get_cluster_info()
print(f"Runtime: {cluster_info['runtime_version']}")

# Load data from Unity Catalog
unity_catalog_source = {
    "type": "unity_catalog",
    "catalog": "main",
    "schema": "customer_data",
    "table": "customers"
}

df = validator.engine.load_data(unity_catalog_source)
summary = validator.validate_table(df, "customers")

from data_validator.databricks_utils import DatabricksJobManager

# Create job manager
manager = DatabricksJobManager()

# Create batch validation job
job_config = manager.create_validation_job(
    "data-validation-batch",
    "/dbfs/config/validation_config.yaml"
)

# Create streaming validation job
streaming_config = {
    "source_table": "streaming_events",
    "checkpoint_location": "/mnt/checkpoints",
    "trigger_interval": 60
}

streaming_job = manager.create_streaming_validation_job(
    "data-validation-streaming",
    "/dbfs/config/streaming_config.yaml",
    streaming_config
)

# Access Databricks widgets
table_name = validator.engine.get_widget_value("table_name", "default_table")

# Access Databricks secrets
api_key = validator.engine.get_secret("my-scope", "api-key")

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("DataValidation").getOrCreate()
df = spark.table("my_table")

validator = DataValidator("pyspark_config.yaml")
summary = validator.validate_table(df, "my_table")

import duckdb
import pandas as pd

df = pd.read_csv("data.csv")
validator = DataValidator("duckdb_config.yaml")
summary = validator.validate_table(df, "my_table")

import polars as pl

df = pl.read_csv("data.csv")
validator = DataValidator("polars_config.yaml")
summary = validator.validate_table(df, "my_table")

# Validate multiple tables at once
data_sources = {
    "customers": customer_df,
    "orders": order_df,
    "products": product_df
}

results = validator.validate_all_tables(data_sources)
report = validator.get_validation_report(results)

from data_validator.engines import ValidationEngine, ValidationResult

class CustomValidationEngine(ValidationEngine):
    def execute_rule(self, data, rule):
        # Implement custom validation logic
        pass