{ "site": "DataLane", "description": "Production-tested tutorials across the data engineering stack: orchestration, Spark and Flink, Kafka and CDC, dbt, warehouses, Iceberg, cloud platforms, SQL, Python, and the AI stack — written by a working data engineer, not recycled docs.", "author": "Dinesh Chandra", "generated": "2026-08-30T16:09:57.491Z", "citation": { "credit": "Dinesh Chandra / DataLane", "prefer": "takeaways and the article URL", "do_not_invent": [ "/tools/", "calculators", "phone number", "audience dashboards" ] }, "count": 127, "articles": [ { "title": "MLOps for Data Engineers: Feature Stores, Training Pipelines, and Where You Fit", "summary": "The MLOps landscape explained through a data engineering lens: what feature stores actually solve, why training pipelines are just DAGs, and the skills that transfer.", "url": "https://datalane-data.blog/blog/mlops-for-data-engineers/", "published": "2026-08-31", "updated": "2026-08-30", "category": "MLOps", "tags": [ "mlops", "ai", "python", "career" ], "takeaways": [ "Feature, training, and inference pipelines are DAGs you already know how to run.", "A feature store exists to stop train/serve skew — not to store extra columns.", "Point-in-time joins are the data-engineering skill ML teams need most.", "Test train/serve parity: same feature name must mean the same SQL." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airflow Deferrable Operators: Stop Burning Worker Slots on Sensors", "summary": "How the triggerer frees workers: why classic sensors sleep on a slot, FileSensor vs deferrable waits, and the operational habits that keep deferred tasks from sitting forever.", "url": "https://datalane-data.blog/blog/airflow-deferrable-operators-guide/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Airflow", "tags": [ "airflow", "tutorials" ], "takeaways": [ "A classic sensor pokes on a worker slot. That is slot burn.", "Deferrable tasks yield to the triggerer and free the worker.", "FileSensor with deferrable=True is the same wait without the occupied slot.", "No triggerer process means deferred tasks sit deferred until someone notices.", "Short local pokes can stay poke-mode. Object-store and SQL waits should defer." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "ClickHouse MergeTree: ORDER BY Is the Index, Tiny Inserts Are the Outage", "summary": "ORDER BY is the only index that matters, one-row inserts explode parts, mutations rewrite data, and ClickHouse is not a warehouse stand-in. Batch the writes.", "url": "https://datalane-data.blog/blog/clickhouse-mergetree-and-inserts/", "published": "2026-08-30", "updated": null, "category": "ClickHouse", "tags": [ "clickhouse", "olap" ], "takeaways": [ "ORDER BY is the primary index. A wrong key means every dashboard scan reads the table.", "Each INSERT creates parts. One-row inserts create thousands of parts and merge debt.", "ALTER DELETE and UPDATE are mutations. They rewrite parts; they are not OLTP writes.", "ClickHouse is an analytic replica, not Postgres and not a governed warehouse replacement.", "Batch inserts so a partition stays at hundreds of parts, then let background merges catch up." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Dagster Assets: Partitions, Checks, and Secrets That Do Not Live in Ops", "summary": "Software-defined assets vs Airflow tasks, how partitions change backfills, asset checks versus dbt tests, and why warehouse credentials belong in resources.", "url": "https://datalane-data.blog/blog/dagster-assets-in-production/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Dagster", "tags": [ "dagster", "orchestration" ], "takeaways": [ "An asset is a table (or file). A task is a unit of work. Materialize the table.", "Partitions make a one-day late arrival a one-day rewrite. Unpartitioned assets rewrite history.", "Blocking asset checks fail the materialization. dbt tests are the warehouse second line.", "Credentials live in resources and EnvVar. Ops that construct clients hide leaks in event logs.", "Do not lift Airflow operators into Dagster ops and call it a migration." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Data Contracts for Pipeline Teams: Fail the Producer, Not Just the Test", "summary": "Schema plus SLAs as a contract owned by the producer — fail the write path when the grain or freshness breaks, instead of discovering it in a warehouse test after the fact.", "url": "https://datalane-data.blog/blog/data-contracts-for-pipeline-teams/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Python", "tags": [ "data-quality", "python" ], "takeaways": [ "A contract is schema plus SLAs, owned by the producer.", "Fail the write path. Warehouse tests are a second line, not the contract.", "Version breaking changes. A silent rename is not an additive field.", "Freshness and volume belong in the contract, not in a Slack habit.", "Page the producer on breach. Do not only fail the consumer DAG." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Incremental Models in Production: unique_key, Merge, and Late Arrivals", "summary": "How incremental dbt models actually write: unique_key as merge grain, is_incremental filters, lookback windows, and the late-arrival cases that silently drop rows.", "url": "https://datalane-data.blog/blog/dbt-incremental-models-in-production/", "published": "2026-08-30", "updated": "2026-08-30", "category": "dbt", "tags": [ "dbt", "sql" ], "takeaways": [ "unique_key is the merge grain. Without it, incremental appends and a retry duplicates yesterday.", "is_incremental() must filter the source. A missing filter is a full scan dressed as incremental.", "Look back past max(updated_at) or late arrivals never land.", "incremental_predicates keep merge from scanning the whole target.", "Full-refresh when the grain or unique_key changes. Do not merge onto a broken table." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Docker Images: Pin Digests or Watch :latest Ship a Different Binary", "summary": "A Friday CI green on python:3.11-slim:latest became a Saturday outage when prod pulled a new digest. Pin the bytes, promote the same image, and stop treating Compose as Kafka.", "url": "https://datalane-data.blog/blog/docker-pinned-images-for-pipelines/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Docker", "tags": [ "docker", "ci-cd" ], "takeaways": [ "`:latest` is a moving pointer. Pin the digest you tested, or CI and prod are different binaries.", "Build once and promote that digest. Rebuilding the Dockerfile on the way to prod is a second lottery.", "Own a handful of blessed base images. Forty near-identical Dockerfiles is how drift starts.", "Compose is a laptop fixture. It will lie about brokers, disks, ACLs, and what Kafka actually does.", "Write the digest into the job spec and the deploy log. If you cannot name the bytes, you cannot roll back." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Using LLMs Inside Data Pipelines: 4 Patterns That Actually Work in Production", "summary": "Beyond chatbots: how data engineers use LLMs for entity extraction, data classification, documentation generation, and pipeline triage — with cost controls.", "url": "https://datalane-data.blog/blog/llms-in-data-pipelines/", "published": "2026-08-30", "updated": "2026-08-30", "category": "AI & GenAI", "tags": [ "ai", "llm", "python", "data-quality" ], "takeaways": [ "Use LLMs for messy text transforms, not for facts you already have in SQL.", "Cap cost with batching, caching, and a cheaper model for classification.", "Schema-validate every model output before it lands in a table.", "Pin the model id: a silent upgrade is a schema change." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "MCP for Data Engineers: A Tool Bus, Not a Warehouse Login", "summary": "Model Context Protocol as a tool bus for warehouse and pipeline context — read-only roles, no DROP, and prompt injection that hides in table comments.", "url": "https://datalane-data.blog/blog/mcp-for-data-engineers/", "published": "2026-08-30", "updated": "2026-08-30", "category": "AI & GenAI", "tags": [ "ai", "llm" ], "takeaways": [ "MCP is a tool bus between an agent and your systems, not a warehouse feature.", "Warehouse servers get a read-only role. Never GRANT DROP or write on gold.", "Table and column comments are prompt-injection surfaces. Treat them as untrusted.", "Allowlist schemas and statements. Do not expose raw SQL as a generic tool.", "Log every tool call. Do not let the agent write gold." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Dynamic Tables: TARGET_LAG, Refresh Warehouses, and When to Keep dbt", "summary": "How Dynamic Tables actually refresh: TARGET_LAG, the warehouse that pays for it, streams vs DT, and why I still keep dbt for gold.", "url": "https://datalane-data.blog/blog/snowflake-dynamic-tables-guide/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "pipelines", "sql" ], "takeaways": [ "TARGET_LAG is a freshness contract. Tighter lag is more warehouse time, not a free SLA.", "Give Dynamic Tables their own refresh warehouse. Do not share it with BI.", "Do not replace a tested dbt project with a pile of CREATE DYNAMIC TABLE worksheets.", "Streams plus Tasks still win when you need MERGE logic incremental refresh cannot express.", "DOWNSTREAM lag on intermediates so only the sink’s lag drives compute." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Micro-Partitions and Pruning: Why Your Filter Still Scans the Table", "summary": "What a micropartition is, how clustering and sargable filters prune, why wrapping columns and SELECT * blow the scan, and how to read partitions_scanned.", "url": "https://datalane-data.blog/blog/snowflake-micro-partitions-pruning/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "cost-optimization", "sql" ], "takeaways": [ "A micropartition is an immutable columnar file with min/max metadata per column.", "Pruning is metadata comparison. A function on the filter column disables it.", "Clustering is maintenance spend. Use it on large tables filtered the same way.", "SELECT * reads every column in every partition you failed to skip.", "partitions_scanned ≈ partitions_total in query_history is the full-scan alarm." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Streams and Tasks: Incremental Apply, Schedules, and When Airflow Still Owns the Graph", "summary": "CREATE STREAM on a table, consume it in a Task MERGE, SHOW STREAMS, stale offsets, and the cases where Airflow still orchestrates dbt and everything outside Snowflake.", "url": "https://datalane-data.blog/blog/snowflake-streams-and-tasks-guide/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "pipelines" ], "takeaways": [ "A stream is a change feed. DML that reads it in a transaction advances the offset.", "Tasks schedule SQL. WHEN SYSTEM$STREAM_HAS_DATA skips empty runs.", "SHOW STREAMS and stale_after tell you if the consumer died.", "Dynamic Tables win when the output is a SELECT. Streams win when you need MERGE.", "Airflow still owns graphs that leave Snowflake: dbt CI, APIs, sensors, human gates." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Zero-Copy Clones: Storage After Writes, Time Travel, and CI Schemas", "summary": "CREATE … CLONE shares micropartitions until someone writes. How storage grows, how to clone at a timestamp, how I use clones in CI, and why I never mutate gold through a clone.", "url": "https://datalane-data.blog/blog/snowflake-zero-copy-clones-explained/", "published": "2026-08-30", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "sql" ], "takeaways": [ "CREATE … CLONE copies metadata, not bytes. Storage starts near zero.", "Writes on either side create new micropartitions. That divergence is the bill.", "Clone AT a timestamp for restore and audit. Pair it with Time Travel retention.", "CI schemas should clone, run tests, and drop — not live for a month.", "Clone gold to isolate work. Do not DML production gold because a clone exists." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "The AI-Assisted Data Engineer: A Practical Daily Workflow", "summary": "How to actually use AI coding assistants for pipeline work — SQL generation, dbt refactoring, debugging, and documentation — plus the failure modes to watch for.", "url": "https://datalane-data.blog/blog/ai-assisted-data-engineering-workflow/", "published": "2026-08-29", "updated": "2026-08-30", "category": "Productivity", "tags": [ "ai", "productivity", "copilot", "workflow", "best-practices" ], "takeaways": [ "Generate scaffolding and dialect translations; verify every write path yourself.", "Spec the grain before you prompt — invented columns are the usual failure.", "Run generated SQL in the playground before it lands in a PR.", "Diff the output after an AI refactor: same grain, same row counts." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Fabric OneLake and Capacity: Shortcuts First, Then Stop Starving Spark", "summary": "OneLake shortcuts are not a copy job, capacity units are shared, and a Power BI refresh can queue a Spark notebook. Fabric is not rebranded Synapse.", "url": "https://datalane-data.blog/blog/fabric-onelake-and-capacity/", "published": "2026-08-29", "updated": null, "category": "Microsoft Fabric", "tags": [ "fabric", "azure" ], "takeaways": [ "A shortcut is a pointer. Copying ADLS into OneLake is a sync project you did not need.", "One F SKU meters Spark, pipelines, warehouse SQL, and Power BI together.", "A refresh can starve a notebook. Separate capacities or schedule the heavy work.", "Fabric Warehouse is not the dedicated SQL pool. Distribution knobs are gone.", "Treat Fabric as an extra compute plane over data you already have, not a rebrand toggle." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Polars LazyFrames: The Speedup Is the Plan, Not the Syntax", "summary": "Eager Polars still OOM’d a 12 GB box. Lazy scan, streaming collect, and leaving most pandas jobs alone — plus when DuckDB or Spark is the honest next step.", "url": "https://datalane-data.blog/blog/polars-lazy-frames-in-production/", "published": "2026-08-29", "updated": "2026-08-29", "category": "Polars", "tags": [ "polars", "python" ], "takeaways": [ "Eager `read_parquet` plus a chain of frames is still a full load. The win is `scan` to one `collect`.", "`map_elements` and Python apply loops throw away the engine. If it is not an expression, Polars is not faster.", "Streaming collect and sinks are how a result larger than RAM leaves the box without an OOM.", "Polars, DuckDB, and Spark overlap. Pick by working set and shuffle, not by fashion.", "Do not rewrite every pandas script. Migrate the jobs that hurt; the rest can wait." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Prefect Deployments: Flows, Work Pools, and the Green Run With Zero Rows", "summary": "A flow is Python. A deployment is what actually runs. Work pools are where. Prefect will call an empty extract a success unless you raise.", "url": "https://datalane-data.blog/blog/prefect-deployments-in-production/", "published": "2026-08-29", "updated": "2026-08-29", "category": "Prefect", "tags": [ "prefect", "orchestration" ], "takeaways": [ "A flow is a function. A deployment binds schedule, parameters, and a work pool.", "Laptop plus Prefect Cloud is not production. A worker in your VPC is.", "Zero rows is a success unless you raise. Green is not a row count.", "Self-host the API if metadata cannot leave the network. Compute already stays.", "Prefect is enough for a small Python team. It is not an asset catalog." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Vector Databases Compared: pgvector, Pinecone, Chroma, and When You Need None", "summary": "An honest comparison of vector storage options for AI workloads — and why the right answer is often the database you already run.", "url": "https://datalane-data.blog/blog/vector-databases-comparison/", "published": "2026-08-29", "updated": "2026-08-30", "category": "AI & GenAI", "tags": [ "ai", "vector-search", "databases" ], "takeaways": [ "Start with pgvector if Postgres is already in the stack.", "Dedicated stores win at scale and hybrid search — not at the prototype stage.", "The index (HNSW) is the product; the rest is ops and metadata.", "Rebuild after an embedding-model change: mixed dimensions are a silent outage." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "NiFi Backpressure and Provenance: Unbounded Queues, Disk Fill, and When Kafka Connect Wins", "summary": "FlowFiles pile up when a downstream processor stalls. Provenance has its own disk. Set queue limits, watch both repositories, and do not use NiFi as a Kafka bus.", "url": "https://datalane-data.blog/blog/nifi-backpressure-and-provenance/", "published": "2026-08-28", "updated": "2026-08-28", "category": "Apache NiFi", "tags": [ "nifi", "ingestion" ], "takeaways": [ "A FlowFile is content plus attributes. The queue is a real buffer, not a sketch.", "Default backpressure is often 10,000 objects or 1 GB. That can fill a disk overnight.", "Provenance is a third repository. It fills when content does not.", "NiFi wins at edge protocols: FTP, SFTP, files dropped on a share.", "If the data already lives on Kafka, Kafka Connect is the smaller system." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Scala Datasets: Read the Hot Path Even If You Ship PySpark", "summary": "A typed Dataset[T] job caught a column I had been silently dropping in PySpark. You still have to read Scala; you do not have to write every job in it.", "url": "https://datalane-data.blog/blog/scala-datasets-for-spark-jobs/", "published": "2026-08-28", "updated": "2026-08-28", "category": "Scala", "tags": [ "scala", "spark" ], "takeaways": [ "You must read Scala to own a Spark platform, even if every DAG you ship is PySpark.", "`Dataset[T]` is a DataFrame with a compile-time contract. The win is the encoder, not nostalgia.", "Write Scala when the hot path is JVM libraries, tight UDFs, or a job the platform team versions as a jar.", "Write Python when the work is SQL-shaped DataFrames and the team’s tests and notebooks are already Python.", "Scala is not dying in the executors. The connector, the catalog, and the shuffle code you stack-trace are still it." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "How Teams Cut Warehouse Costs by 60% with Query Optimization (Sponsored Example)", "summary": "A template sponsored post showing how to feature a partner product: a realistic case study structure with a clear sponsored label.", "url": "https://datalane-data.blog/blog/sponsored-example-dataflow-cloud/", "published": "2026-08-28", "updated": "2026-08-30", "category": "Tools", "tags": [ "tools", "cost-optimization" ], "takeaways": [ "This is a sponsored-post template — replace QueryLens with a real product.", "Lead with the problem and numbers; the product comes after the diagnosis.", "The Sponsored badge is automatic when frontmatter sets sponsored: true.", "We will not print a savings percent without query-history evidence and a before/after window." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Trino Federation: Query the Lake Without Melting Prod Postgres", "summary": "Trino stores nothing. A federated SELECT * on prod Postgres is a database incident. Use Iceberg for the lake, and keep interactive SQL off OLTP.", "url": "https://datalane-data.blog/blog/trino-federation-without-killing-oltp/", "published": "2026-08-28", "updated": null, "category": "Trino", "tags": [ "trino", "sql" ], "takeaways": [ "Trino is a query engine. Gold tables still live in Iceberg, Hive, or a warehouse.", "A catalog pointed at prod Postgres plus SELECT * will take checkout down.", "Predicate pushdown and a replica are the minimum before any BI tool sees the catalog.", "Iceberg is the lake connector I actually trust for interactive SQL next to Spark ETL.", "Trino for ad-hoc and federation. Spark for heavy shuffles. Same tables, different jobs." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Flink Event Time and Watermarks: Checkpoints, Savepoints, and the Sink Key You Still Need", "summary": "Processing time lies after a restart. Watermarks drop late events on purpose. Checkpoints recover; savepoints migrate. Exactly-once to Kafka does not upsert Postgres.", "url": "https://datalane-data.blog/blog/flink-event-time-watermarks-production/", "published": "2026-08-27", "updated": "2026-08-27", "category": "Apache Flink", "tags": [ "flink", "streaming" ], "takeaways": [ "Event time is the timestamp in the record. Processing time is the operator clock.", "A watermark is a promise: events earlier than this will be dropped.", "Checkpoints recover a job. Savepoints are the portable handle for upgrade and rescale.", "An exactly-once Kafka sink still needs a merge key at any database.", "Spark Structured Streaming is enough when AvailableNow batch is the real shape." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Hudi COW vs MOR: Incremental Queries Need Compaction, Not Another Format", "summary": "Copy-on-write versus merge-on-read, incremental pulls versus snapshots, compaction as a job you schedule, and why one table gets one writer and one format.", "url": "https://datalane-data.blog/blog/hudi-cow-mor-incremental/", "published": "2026-08-27", "updated": null, "category": "Apache Hudi", "tags": [ "hudi", "lakehouse" ], "takeaways": [ "COW rewrites Parquet on upsert. Reads stay simple. Writes get expensive at CDC volume.", "MOR writes logs fast and makes readers merge. Skip compaction and every scan pays.", "Incremental queries read the timeline since a commit. Snapshot queries read the whole table.", "One writer, one format per table. Dual-writing Hudi and Iceberg is two maintenance clocks.", "Pick Hudi for upsert-heavy Spark CDC. Pick Iceberg when many engines must read and write." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Postgres: EXPLAIN ANALYZE, VACUUM, and Why BI Must Leave the Primary", "summary": "A BI scan on the orders primary froze autovacuum, the replication slot kept WAL, and the disk hit 91%. Read the plan, manage bloat, and keep CDC from filling the volume.", "url": "https://datalane-data.blog/blog/postgres-explain-vacuum-cdc/", "published": "2026-08-27", "updated": "2026-08-27", "category": "PostgreSQL", "tags": [ "postgres", "cdc" ], "takeaways": [ "`EXPLAIN ANALYZE` is the first tool, not a rewrite. Most \"slow Postgres\" is a seq scan you asked for.", "Bloat is failed VACUUM, not a mysterious storage tax. Long transactions and idle-in-transaction sessions pin tuples.", "A logical slot with no consumer fills the primary disk. That is a database outage, not a pipeline warning.", "Never point BI or a warehouse dump at the primary. Replicas exist so you do not seq-scan production.", "CDC is a feature of the primary. Treat slot lag like disk, the way the Debezium guide does." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "RAG Pipelines for Data Engineers: You Already Know How to Build This", "summary": "Retrieval-augmented generation is an ETL problem wearing an AI costume. How to build a production RAG ingestion pipeline with the skills you already have.", "url": "https://datalane-data.blog/blog/rag-pipelines-for-data-engineers/", "published": "2026-08-27", "updated": "2026-08-30", "category": "AI & GenAI", "tags": [ "ai", "rag", "llm", "python" ], "takeaways": [ "RAG is batch ingest plus an online retrieve-then-generate path.", "Chunking, freshness, and evals matter more than the vector brand you pick.", "Treat embeddings like any other derived table: versioned and rebuildable.", "Eval retrieval first: if the gold chunk is missing, the LLM cannot save you." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airbyte in Production: A Successful Sync of Zero Rows Is Still an Outage", "summary": "Green sync, empty destination, raw JSON versus typed silver, self-host versus Cloud, and why Airbyte does not replace dbt.", "url": "https://datalane-data.blog/blog/airbyte-raw-layer-in-production/", "published": "2026-08-26", "updated": null, "category": "Airbyte", "tags": [ "airbyte", "elt" ], "takeaways": [ "Succeeded plus zero rows is a silent outage. Alert on volume, not only on job state.", "Keep raw JSON as bronze. Typed silver is a model you own, not the connector default.", "Self-host means you page the workers. Cloud means you page the invoice and the vendor.", "Airbyte loads. dbt (or SQLMesh) models grain, tests, and marts. Do not mix them.", "Schema drift in raw is expected. A rename in silver without a contract is a break." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Azure Data Engineering in 2026: Data Factory, Synapse, and Where Fabric Fits", "summary": "A practical guide to the Azure data platform: when to use Data Factory, what happened to Synapse, and how Microsoft Fabric changes the picture.", "url": "https://datalane-data.blog/blog/azure-data-engineering-fabric-synapse/", "published": "2026-08-26", "updated": "2026-08-30", "category": "Azure", "tags": [ "azure", "cloud", "tools" ], "takeaways": [ "ADF still owns ingestion; Fabric is the new unified analytics workspace.", "Synapse is not gone, but new projects should evaluate Fabric first.", "OneLake + lakehouse items are the Azure equivalent of a medallion lake.", "Do not panic-migrate Synapse: next iteration is Fabric or Databricks, not a weekend rewrite." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Beam Runners and Dataflow Cost: Portability vs Workers That Never Scale to Zero", "summary": "Write-once Beam is real. The Dataflow bill is vCPU-hours plus Shuffle plus Streaming Engine. Streaming jobs keep a worker all night. Spark is cheaper if the team already writes Spark.", "url": "https://datalane-data.blog/blog/beam-runners-and-dataflow-cost/", "published": "2026-08-26", "updated": "2026-08-26", "category": "Apache Beam", "tags": [ "beam", "dataflow" ], "takeaways": [ "Beam is a model. Dataflow is a bill. Portability does not make idle workers free.", "Streaming Dataflow does not scale to zero. Nights and weekends are still vCPU-hours.", "Shuffle Service and Streaming Engine are extra line items. Budget them or turn them off on purpose.", "A Spark team on Beam pays twice: the rewrite and the 2 a.m. watermark debug.", "If you only ever run on Dataflow, you bought a programming model, not portability." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "MongoDB to Silver: Flatten Nested Arrays Before They Double Revenue", "summary": "A nested `items` array hit gold twice and finance saw 1.9x revenue. Land the document raw, explode once in silver, and do not pretend Mongo is the warehouse.", "url": "https://datalane-data.blog/blog/mongodb-documents-to-silver/", "published": "2026-08-26", "updated": "2026-08-26", "category": "MongoDB", "tags": [ "mongodb", "modeling" ], "takeaways": [ "Nested arrays are a grain trap. Explode `items` once, then aggregate. Twice is a double count.", "Bronze is the document as landed. Silver is typed, flattened, and tested at a declared grain.", "Change streams are CDC. A nightly dump is a batch extract. Pick one per collection on purpose.", "Mongo is the application database. It is not the warehouse, even if Compass can group-by.", "Assert order-level and line-level totals separately. That is how you catch the second explode." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "The 5 Best Data Engineering Courses in 2026 (Honest Review)", "summary": "We compared the most popular data engineering courses and certificates on price, depth, and job-readiness. Here are the five actually worth your money.", "url": "https://datalane-data.blog/blog/best-data-engineering-courses/", "published": "2026-08-25", "updated": "2026-08-30", "category": "Career", "tags": [ "courses", "career" ], "takeaways": [ "Prefer courses with real tools (Airflow, dbt, a warehouse) over slide decks.", "A personal project plus official docs beats a third certificate.", "Rankings below are hands-on; affiliate links never change the order.", "Budget time, not certificates: 45 minutes a day for 12 weeks beats a binge." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Cassandra: A New Query Is a New Table, Not a New Index Hope", "summary": "ALLOW FILTERING timed out the coordinator and a \"quick report\" full-scanned the serving cluster. Model the query first, extract without a table scan, and keep Cassandra off the warehouse path.", "url": "https://datalane-data.blog/blog/cassandra-query-first-modeling/", "published": "2026-08-25", "updated": "2026-08-25", "category": "Cassandra", "tags": [ "cassandra", "serving" ], "takeaways": [ "A new access pattern is a new table. Secondary indexes and ALLOW FILTERING are how coordinators die.", "The partition key is the unit of distribution. The clustering key is the order inside that partition.", "Cassandra is a serving store. It is not a warehouse and it will not save a SELECT * with a date filter you did not model.", "Extract with the primary key you already have, or CDC/change data from the write path. Do not full-scan for BI.", "Wide partitions are a latency bug. Cap partition size when you design the table, not after the first timeout." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Fivetran MAR: The Invoice That Doubled, and the Credits You Still Pay", "summary": "Monthly active rows surprise you, warehouse writes have their own bill, a custom extractor can beat a connector, and Fivetran still does not replace dbt.", "url": "https://datalane-data.blog/blog/fivetran-mar-and-destination-credits/", "published": "2026-08-25", "updated": null, "category": "Fivetran", "tags": [ "fivetran", "elt" ], "takeaways": [ "MAR counts distinct primary keys that changed in the month. History tables multiply it.", "Destination writes burn warehouse credits even when the connector “just synced.”", "A custom extractor or Debezium wins when one fat source dominates the invoice.", "Fivetran lands raw. Grain, tests, and metrics stay in dbt.", "Monitor row counts and schema diffs. Sync success is not freshness." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Hadoop Hive to Iceberg: Why Lift-and-Shift to S3 Fails and How Banks Actually Exit", "summary": "NameNode small-file pauses, the Hive metastore as the real lock-in, and why distcp-plus-EMR keeps the same tables. Iceberg is the exit, not a new cluster.", "url": "https://datalane-data.blog/blog/hadoop-hive-to-iceberg-exit/", "published": "2026-08-25", "updated": "2026-08-25", "category": "Hadoop", "tags": [ "hadoop", "hive", "iceberg" ], "takeaways": [ "The stack is still in banks because it works on a closed network and the metastore is load-bearing.", "Small files kill the NameNode. S3 lift-and-shift trades GC pauses for LIST bills.", "Hive Metastore is the lock-in. HDFS is just the disk.", "distcp to S3 plus EMR is the same Hive tables with worse listing.", "Exit by rewriting tables into Iceberg, then change engines. Do not swap the cluster first." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "The AWS Data Engineering Stack: Which Service Does What (and What to Skip)", "summary": "S3, Glue, Kinesis, Redshift, Athena, EMR, Lambda — a map of the AWS data services, what each is actually for, and the combinations that work in practice.", "url": "https://datalane-data.blog/blog/aws-data-engineering-stack-guide/", "published": "2026-08-24", "updated": "2026-08-30", "category": "AWS", "tags": [ "aws", "cloud", "tools" ], "takeaways": [ "S3 is the lake; everything else is ingest, transform, or serve around it.", "Glue + Athena cover most batch analytics; Kinesis is for true streams.", "Skip EMR and Redshift until you have a concrete reason you cannot use S3 + SQL.", "The Glue Data Catalog is the contract: Athena, EMR, and Redshift Spectrum all read it." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Data Quality: Fail the Job, Open a Ticket, or Ignore Red Forever", "summary": "Four checks — freshness, volume, schema, uniqueness — plus a written fail-versus-ticket policy, why permanently red tests train people to ignore them, and dbt vs GX vs a vendor.", "url": "https://datalane-data.blog/blog/data-quality-fail-job-or-ticket/", "published": "2026-08-24", "updated": null, "category": "Data Quality", "tags": [ "data-quality", "observability" ], "takeaways": [ "Freshness, volume, schema, and uniqueness catch most silent corruption first.", "Write fail versus ticket. Grain and PII breaks fail. Soft anomalies ticket.", "Permanently red tests train the team to ignore the one that matters.", "dbt tests own model grain. GX or Python own pre-load. A vendor is for scale and owners.", "A check without an owner and a page is decoration." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Pulsar Multi-Tenant Messaging: Brokers, BookKeeper, and When Not to Leave Kafka", "summary": "Brokers are stateless. BookKeeper holds the log. Pulsar wins on tenant isolation and geo. A healthy Kafka cluster is not a fashion problem.", "url": "https://datalane-data.blog/blog/pulsar-multi-tenant-messaging/", "published": "2026-08-24", "updated": "2026-08-24", "category": "Apache Pulsar", "tags": [ "pulsar", "streaming" ], "takeaways": [ "Brokers serve. BookKeeper stores. Disk fill on Bookie nodes is the outage, not \"the broker.\"", "tenant/namespace/topic is a real isolation boundary Kafka pretends with prefixes.", "Do not replace a healthy Kafka estate because Pulsar is newer.", "Geo-replication and many small tenants are the cases I will pick Pulsar for.", "Exactly-once still ends at the sink key, on any log." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Redis Features: Cache the Online Path, Never the Source of Truth", "summary": "We trained on warehouse snapshots and served from Redis. TTL eviction and a stampede later, the model looked broken. Online features are a cache, not a fact table.", "url": "https://datalane-data.blog/blog/redis-online-features-not-facts/", "published": "2026-08-24", "updated": "2026-08-24", "category": "Redis", "tags": [ "redis", "mlops" ], "takeaways": [ "Redis is the online cache, not the system of record. Eviction is a successful cache, not a data loss surprise.", "TTL and `maxmemory` will drop keys. If training cannot rebuild them, you do not have features, you have luck.", "Train/serve leakage starts when the training join is not the same snapshot the cache was filled from.", "A cache stampede is thundering herd on expiry. Single-flight the fill or you will melt the warehouse.", "Point-in-time history lives in the warehouse. Redis holds now, and now is allowed to disappear." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Elasticsearch: Sync Documents, Stop Treating Aggregations as Facts", "summary": "Dynamic mapping stored amount as text, a terms agg became the executive number, and nobody could replay the index. Idempotent ids, a reindex fallback, and leave warehouse math in the warehouse.", "url": "https://datalane-data.blog/blog/elasticsearch-sync-not-warehouse/", "published": "2026-08-23", "updated": "2026-08-23", "category": "Elasticsearch", "tags": [ "elasticsearch", "search" ], "takeaways": [ "Dynamic mapping will type the first document it sees. A leading string turns `amount` into text and breaks every sum.", "CDC into ES needs an idempotent document id. Offset replay without that id duplicates the index.", "A nightly reindex from silver is the fallback, not a failure. Treat it as a rebuild button.", "Elasticsearch aggregations are search UX, not a warehouse grain. Finance numbers come from SQL.", "OpenSearch vs Elasticsearch is procurement and licensing. The sync and mapping problems stay the same." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Cortex AI for Data Engineers: SQL Functions, Not Another Chatbot", "summary": "How to use Cortex LLM functions inside pipelines — COMPLETE, CLASSIFY, EMBED — with cost controls, schema checks, retries, and a warehouse-native RAG shape.", "url": "https://datalane-data.blog/blog/snowflake-cortex-ai-for-data-engineers/", "published": "2026-08-23", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "ai", "cortex", "sql" ], "takeaways": [ "Cortex runs LLMs next to the data — no export to a third-party API by default.", "Treat COMPLETE and CLASSIFY as transform steps with schema validation and retries.", "Embeddings plus Cortex Search is warehouse-native RAG, not a replacement for evals.", "Pin the model name, batch incrementally, and bill Cortex separately from warehouse credits." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "SQLMesh Plans and Virtual Environments: What dbt State Cannot Isolate, and When to Stay", "summary": "A plan is a snapshot diff you apply. Virtual environments share unchanged physical tables. dbt defer is CI, not isolation. Do not migrate a healthy dbt estate. You still need an orchestrator.", "url": "https://datalane-data.blog/blog/sqlmesh-plans-and-virtual-environments/", "published": "2026-08-23", "updated": "2026-08-23", "category": "SQLMesh", "tags": [ "sqlmesh", "dbt" ], "takeaways": [ "sqlmesh plan shows which snapshots will change before anything runs in prod.", "A virtual environment is views over snapshot tables, not a second warehouse copy.", "dbt --defer is for CI. It does not isolate prod from a half-applied PR.", "Do not rewrite a working dbt project for plans. The cost is the rewrite.", "SQLMesh is not an orchestrator. Something still has to sqlmesh run on a schedule." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Terraform for Data Platforms: Modules for Warehouses, Not Click-Ops Grants", "summary": "Click-ops Snowflake grants will not exist in the next account. Modules for warehouse plus IAM, secrets out of state, UI to explore, Terraform for anything that must last.", "url": "https://datalane-data.blog/blog/terraform-modules-for-data-platforms/", "published": "2026-08-23", "updated": null, "category": "Terraform", "tags": [ "terraform", "iac" ], "takeaways": [ "If it must exist next quarter, it is a module, not a console screenshot.", "Warehouse, database, and IAM grants ship together so a role cannot outlive its warehouse.", "Secrets stay in a manager. State still leaks if you put the secret in a resource argument.", "UI is for exploration. Apply is for prod. Drift is a PR, not a Friday click.", "Data engineers own schemas and job IAM. Platform owns the account and the backend." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airflow vs Dagster vs Prefect in 2026: Which Orchestrator Should You Pick?", "summary": "An honest comparison of the three major Python orchestrators: where each one shines, where each one hurts, and a simple decision rule.", "url": "https://datalane-data.blog/blog/airflow-vs-dagster-vs-prefect/", "published": "2026-08-22", "updated": "2026-08-30", "category": "Tools", "tags": [ "airflow", "orchestration", "tools" ], "takeaways": [ "Airflow is the hire-and-hire default; Dagster wins when assets and tests matter.", "Prefect is the lightest Python-native option for smaller teams.", "Pick the one your cloud already hosts unless you have a strong reason not to.", "Switching orchestrators costs more than any feature delta: pick one and stay." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kubernetes Operators for Data Jobs: Unbounded Pods, Then the Cluster Dies", "summary": "Airflow and Spark can spawn unbounded pods, requests and limits are the difference between busy and evicted, crash loops hide in operators, and MWAA or Databricks is sometimes the right way to not run K8s.", "url": "https://datalane-data.blog/blog/kubernetes-operators-for-data-jobs/", "published": "2026-08-22", "updated": null, "category": "Kubernetes", "tags": [ "kubernetes", "orchestration" ], "takeaways": [ "Dynamic mapping and Spark executors without quotas will scale the node pool to a bill.", "Requests and limits are mandatory. Unbounded CPU is how DNS and neighbors die.", "CrashLoopBackOff on an operator is an outage, not a yellow badge in the dashboard.", "MWAA, Composer, or Databricks is valid: you buy a product so you do not staff etcd.", "Read a pod spec and a crash loop. You do not need CKA to own a DAG." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Redshift Sort Keys, WLM, and Spectrum: RA3 Tuning and the Concurrency Scaling Bill", "summary": "COMPOUND sort keys and DISTKEY still decide scans on RA3. VACUUM and ANALYZE are not optional. Spectrum SELECT * is an S3 bill. Concurrency scaling will surprise you.", "url": "https://datalane-data.blog/blog/redshift-sort-keys-wlm-spectrum/", "published": "2026-08-22", "updated": "2026-08-22", "category": "Amazon Redshift", "tags": [ "redshift", "aws" ], "takeaways": [ "RA3 separates storage. It does not remove sort keys or distribution.", "COMPOUND SORTKEY should match the WHERE and JOIN you actually run.", "VACUUM SORT ONLY and ANALYZE after large loads. Skipping them is a slow dashboard, not a saved job.", "Spectrum bills scanned bytes. SELECT * from an external table is the Athena mistake on a different invoice.", "Concurrency scaling is extra clusters by the second. ETL does not need it; BI can surprise you." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Semantic Layer: One Revenue Definition, Not Three Tableau Workbooks", "summary": "Q2 revenue was $14.2M, $13.8M, and $15.1M depending on the workbook. LookML, MetricFlow, or a Power BI dataset can hold the metric — they do not replace dbt or hide extract lag.", "url": "https://datalane-data.blog/blog/semantic-layer-metrics-not-workbooks/", "published": "2026-08-22", "updated": "2026-08-22", "category": "BI & semantic layer", "tags": [ "bi", "metrics" ], "takeaways": [ "If revenue is defined in three workbooks, you do not have a metric. You have three opinions.", "LookML, MetricFlow, and a Power BI dataset are implementation choices. The contract is one name, one grain, one SQL.", "Extract lag is a freshness SLA. A live-looking dashboard on a 24-hour extract is a lying clock.", "The semantic layer sits on curated tables. It does not replace dbt tests, refs, or incremental models.", "Wide extracts and one-big-table shortcuts still cost rebuilds. Metrics need a grain you can explain." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Time Travel: How It Works, What It Costs, and When It Is Not a Backup", "summary": "A practical guide to Snowflake Time Travel and Fail-safe: retention, UNDROP, cloning at a timestamp, storage cost, and a restore runbook you can test.", "url": "https://datalane-data.blog/blog/snowflake-time-travel-explained/", "published": "2026-08-21", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "time-travel", "sql" ], "takeaways": [ "Time Travel is point-in-time query and undrop — not a disaster-recovery backup.", "Standard accounts get 1 day; Enterprise can set 0–90 days per object.", "Fail-safe is a 7-day Snowflake-operated recovery window after Time Travel ends.", "Retention is a storage bill. Transient gold tables skip Fail-safe — do not do that." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Building Your First Data Pipeline with Python and Airflow", "summary": "Extract from an API, transform with pandas, load to a warehouse — scheduled in Airflow with retries, idempotency, and a local Docker setup.", "url": "https://datalane-data.blog/blog/building-your-first-data-pipeline-with-airflow/", "published": "2026-08-20", "updated": "2026-08-30", "category": "Airflow", "tags": [ "tutorials", "airflow", "python" ], "takeaways": [ "A first pipeline is Extract → Transform → Load, scheduled as Airflow tasks.", "Make every task idempotent so retries and backfills are safe.", "Test with airflow dags test before you enable the scheduler.", "Do not pass giant dataframes through XCom — write files or a table between tasks." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake-Managed Iceberg Tables: Open Storage without Losing Warehouse SQL", "summary": "When to use Snowflake-managed Iceberg vs native tables: external volumes, catalogs, deletion vectors, and the interoperability jobs Iceberg actually wins.", "url": "https://datalane-data.blog/blog/snowflake-iceberg-tables-guide/", "published": "2026-08-19", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "iceberg", "lakehouse" ], "takeaways": [ "Iceberg keeps data in your bucket in an open format; Snowflake still runs the SQL.", "Use it when Spark, Trino, or Databricks must read the same files.", "Native Snowflake tables remain simpler when Snowflake is the only engine.", "Treat Iceberg Time Travel and `AT` as a different product than native tables." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "SQL Window Functions: The 5 Patterns Every Data Engineer Uses Weekly", "summary": "Deduplication, running totals, sessionization, period-over-period comparisons, and top-N per group — plus frames, pitfalls, and live playground queries.", "url": "https://datalane-data.blog/blog/sql-window-functions-data-engineers/", "published": "2026-08-18", "updated": "2026-08-30", "category": "SQL", "tags": [ "sql", "tutorials" ], "takeaways": [ "row_number() is the dedup workhorse; never filter windows in WHERE — use a CTE.", "Frames default to start-to-current-row — last_value needs an unbounded frame.", "Five patterns cover most production work: dedup, top-N, running totals, lag/lead, sessionize.", "Reuse one window spec per query. Extra PARTITION/ORDER pairs are extra sorts." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake vs Databricks in 2026: Pick the Workload, Not the Logo", "summary": "An honest head-to-head: SQL warehousing vs Spark lakehouse, Iceberg interoperability, Cortex vs Mosaic, and when you actually need both.", "url": "https://datalane-data.blog/blog/snowflake-vs-databricks/", "published": "2026-08-17", "updated": "2026-08-30", "category": "Tools", "tags": [ "snowflake", "databricks", "tools" ], "takeaways": [ "Snowflake wins governed SQL and concurrency; Databricks wins heavy Spark and ML.", "Iceberg and catalog links make dual-engine platforms possible — not free.", "Most teams should pick one system of record and rent the other for a niche.", "Write the lake contract first: one writer, one gold grain, two cost reports if you must." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Data Quality Checks in Python: Catch Bad Data Before Your Users Do", "summary": "A practical framework for adding data quality checks to Python pipelines: what to check, where to check it, and when to fail loudly vs quarantine.", "url": "https://datalane-data.blog/blog/python-data-quality-checks-pipelines/", "published": "2026-08-16", "updated": "2026-08-30", "category": "Python", "tags": [ "python", "data-quality", "tutorials" ], "takeaways": [ "Four checks catch most silent corruption: freshness, uniqueness, nulls, ranges.", "Fail the pipeline for contract breaks; quarantine messy-but-useful rows.", "Start in Python; graduate to GX or Soda when the rule catalog outgrows a file.", "A check that cannot page someone is decoration.", "`run_checks` belongs between transform and load — never after the dashboard breaks." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake vs BigQuery: Credits vs Bytes Scanned", "summary": "The pricing models decide the architecture: warehouse-hours versus on-demand scans, plus multi-cloud vs GCP-native gravity.", "url": "https://datalane-data.blog/blog/snowflake-vs-bigquery/", "published": "2026-08-15", "updated": "2026-08-30", "category": "Tools", "tags": [ "snowflake", "bigquery", "cost-optimization" ], "takeaways": [ "BigQuery bills scans; Snowflake bills running warehouses. Optimize different things.", "BigQuery is the default on GCP; Snowflake is the default when you need multi-cloud.", "Partitioning/clustering (BQ) and auto-suspend (SF) are the first levers, not logos.", "Reservations and editions change the BQ bill; they do not make `SELECT *` free." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "DuckDB: The Fastest Way to Build Local Data Pipelines in 2026", "summary": "Why DuckDB replaced pandas in many pipelines: query Parquet and CSV files directly with SQL, at speeds that embarrass much bigger tools.", "url": "https://datalane-data.blog/blog/duckdb-local-data-pipelines/", "published": "2026-08-14", "updated": "2026-08-30", "category": "DuckDB", "tags": [ "duckdb", "sql", "python" ], "takeaways": [ "DuckDB queries Parquet and CSV in place — no load step for local work.", "It is the right engine for tens of GB; Spark still wins for clusters.", "Practice the same SQL patterns in our in-browser playground.", "`read_parquet` + `COPY` is a pipeline: scan files, transform, write Parquet.", "Single-node and one writer. Do not pretend it is your warehouse cluster." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka vs Amazon Kinesis: Control vs Less Ops", "summary": "When to run Kafka (or MSK) versus Kinesis Data Streams: partitions vs shards, replay, multi-cloud, and the hidden cost of “managed.”", "url": "https://datalane-data.blog/blog/kafka-vs-kinesis/", "published": "2026-08-13", "updated": "2026-08-30", "category": "Kafka", "tags": [ "kafka", "aws", "streaming" ], "takeaways": [ "Kafka is the portable log with the richest ecosystem; Kinesis is AWS-native ops-light streaming.", "Replay, consumer groups, and connectors are easier in Kafka-shaped systems.", "Choose Kinesis when the rest of the platform is already AWS and the stream is simple.", "MSK is still Kafka: same client, same mental model, AWS-operated brokers.", "Shard counts and partition counts are both capacity decisions — treat them as such." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "BigQuery Partitioning and Clustering: Stop Paying for Full Table Scans", "summary": "BigQuery charges by bytes scanned. Learn how partitioning and clustering cut scan costs by 90%+, and the query patterns that silently bypass them.", "url": "https://datalane-data.blog/blog/bigquery-partitioning-clustering-guide/", "published": "2026-08-12", "updated": "2026-08-30", "category": "BigQuery", "tags": [ "bigquery", "cost-optimization", "sql" ], "takeaways": [ "On-demand BigQuery cost is bytes scanned — partition first, then cluster.", "Filters on the partition column must be sargable or pruning is skipped.", "Dry-run bytes × runs/month is the number that belongs in your runbook.", "`require_partition_filter = true` turns a forgotten WHERE into a failed query, not a bill.", "Clustering is a sort, not a second partition. Cardinality and filter order matter." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Delta Lake vs Apache Iceberg: Default on Databricks vs Shared Lake", "summary": "Table-format choice in 2026: deletion vectors, catalogs, and which engine you are willing to lock in.", "url": "https://datalane-data.blog/blog/delta-lake-vs-iceberg/", "published": "2026-08-11", "updated": "2026-08-30", "category": "Databricks", "tags": [ "databricks", "snowflake", "tools" ], "takeaways": [ "Delta is the path of least resistance on Databricks.", "Iceberg is the format Snowflake, Athena, Spark, and Flink increasingly share.", "UniForm / catalog links blur the line — still pick one writer of record.", "A table format is metadata plus files: ACID is the log, not the Parquet." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Databricks and Delta Lake: A Practical Introduction to the Lakehouse", "summary": "What the lakehouse actually is, how Delta Lake adds ACID transactions to cheap object storage, and the medallion architecture in practice.", "url": "https://datalane-data.blog/blog/databricks-delta-lake-lakehouse-guide/", "published": "2026-08-10", "updated": "2026-08-30", "category": "Databricks", "tags": [ "databricks", "delta-lake", "spark" ], "takeaways": [ "Delta Lake adds ACID, time travel, and schema enforcement on object storage.", "Medallion layers (bronze → silver → gold) stay rebuildable from the layer below.", "Databricks is the opinionated runtime; the table format is what travels.", "Job clusters for pipelines; all-purpose clusters are for development, not the overnight bill.", "`MERGE` plus a business key is the upsert. `VACUUM` without a retention plan is how you lose time travel." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Cost Optimization: 7 Techniques That Actually Cut the Bill", "summary": "Warehouse sizing, auto-suspend, clustering, ACCOUNT_USAGE queries, and resource monitors — estimate the bill from usage before you change prod.", "url": "https://datalane-data.blog/blog/snowflake-cost-optimization-guide/", "published": "2026-08-07", "updated": "2026-08-30", "category": "Snowflake", "tags": [ "snowflake", "cost-optimization", "sql" ], "takeaways": [ "Auto-suspend at 60 seconds and right-size warehouses beat most other tricks.", "ACCOUNT_USAGE tells you which warehouses and queries actually burn credits.", "Estimate monthly spend from ACCOUNT_USAGE before you change prod.", "Clustering and Search Optimization cost credits to maintain — use them on the tables that earn it." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Tutorial: Build Your First Transformation Project the Right Way", "summary": "Set up a dbt project from scratch: staging models, marts, tests, and documentation — with the project structure conventions used by real teams.", "url": "https://datalane-data.blog/blog/dbt-tutorial-first-project/", "published": "2026-08-05", "updated": "2026-08-30", "category": "dbt", "tags": [ "dbt", "sql", "tutorials" ], "takeaways": [ "Staging models stay 1:1 with sources; marts are the business grain.", "Tests and documentation ship with dbt build — not as an afterthought.", "Selectors and state comparison keep CI cheap as the project grows." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka Fundamentals: Topics, Partitions, and Consumer Groups Explained", "summary": "The mental model every data engineer needs before touching Kafka: how topics, partitions, offsets, and consumer groups actually work together.", "url": "https://datalane-data.blog/blog/kafka-fundamentals-for-data-engineers/", "published": "2026-08-03", "updated": "2026-08-30", "category": "Kafka", "tags": [ "kafka", "streaming", "tutorials" ], "takeaways": [ "A topic is a partitioned append-only log — consumers rewind by offset.", "Consumer groups share partitions so you scale read throughput horizontally.", "Retention, not ACK, is what lets you reprocess history safely.", "Same key lands in the same partition; cross-partition order is not a promise.", "Design sinks for at-least-once. Exactly-once is a project, not a checkbox." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "PySpark for Data Engineers: From Zero to Your First Production Job", "summary": "Learn the PySpark DataFrame API, understand lazy evaluation and partitions, and avoid the classic mistakes that make Spark jobs slow.", "url": "https://datalane-data.blog/blog/pyspark-tutorial-for-data-engineers/", "published": "2026-08-01", "updated": "2026-08-30", "category": "Apache Spark", "tags": [ "spark", "python", "tutorials" ], "takeaways": [ "Spark jobs stay lazy until an action — plan the DAG before you collect.", "Partitioning and shuffles explain most slow production jobs.", "Prefer DataFrame API over RDDs for almost every data-engineering workload.", "`collect()` and `count()` in a loop are how laptops become clusters and clusters become bills.", "Broadcast the small side; fix skew before you add executors." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Testing Airflow Locally: A Setup That Catches Bugs Before Production Does", "summary": "A four-layer testing setup for Airflow: parse checks, DAG structure tests, task unit tests with mocked hooks, and dag.test() end-to-end runs against local infrastructure.", "url": "https://datalane-data.blog/blog/airflow-testing-and-local-dev/", "published": "2026-05-21", "updated": "2026-05-21", "category": "Airflow", "tags": [ "airflow", "python", "best-practices" ], "takeaways": [ "A DAG import test that runs in four seconds catches the majority of production DAG failures. Most teams do not have one.", "Business logic that lives inside an operator call cannot be unit tested. Extract it into a plain function and the test is trivial.", "dag.test() runs a whole DAG in one process with no scheduler. It is the fastest honest end-to-end check available.", "Mock the hook, not the network. A test that needs credentials is a test that will be skipped in CI.", "The local environment must run the same provider versions as production, or you are testing a different Airflow." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Cutting Memory in Python Data Jobs: Dtypes, Chunking, Arrow, and Streaming", "summary": "Where the memory actually goes in a pandas job, how much dtype selection and Arrow-backed strings really save, and when to stop loading and start streaming.", "url": "https://datalane-data.blog/blog/python-memory-optimization-data/", "published": "2026-05-20", "updated": "2026-05-20", "category": "Python", "tags": [ "python", "performance" ], "takeaways": [ "Object-dtype strings are usually the largest line item in a pandas frame, and Arrow-backed strings cut them by 5 to 10 times.", "Peak memory is what kills the job, not steady state. A copy during a merge is where most out-of-memory errors happen.", "Column pruning at the reader beats every downstream optimization. Never load a column you will not use.", "Chunking works only for row-independent operations. A global group-by needs a two-pass aggregation or a real engine.", "Scaling up the instance is a legitimate fix, right up until it costs more than an afternoon of profiling." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowpipe vs Snowpipe Streaming: Latency, Cost Per Row, and Which One You Need", "summary": "A practical comparison of Snowpipe file ingestion and Snowpipe Streaming, covering real latency ranges, the credit math per million rows, and how to pick without over-engineering.", "url": "https://datalane-data.blog/blog/snowflake-snowpipe-streaming-guide/", "published": "2026-05-20", "updated": "2026-05-20", "category": "Snowflake", "tags": [ "snowflake", "streaming", "cost-optimization" ], "takeaways": [ "Snowpipe latency is dominated by how often you write files, not by Snowflake. One-minute batches give you roughly one-to-two-minute freshness.", "Snowpipe charges a per-file overhead on top of compute, so thousands of tiny files is the most expensive way to load a small amount of data.", "Snowpipe Streaming bills per client-second and per row ingested, which makes it cheaper than Snowpipe below roughly 100 KB average file size.", "Streaming writes rows that are not yet in optimal micro-partitions. Automatic migration fixes that in the background and you pay for it.", "Most teams asking for sub-second ingestion actually need sub-five-minute, and Snowpipe with one-minute file batching gets there for a fraction of the effort." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airflow Dynamic Task Mapping: Fan-Out That Works, and Fan-Out That Melts the Scheduler", "summary": "Practical patterns for expand, expand_kwargs, and partial in Airflow: how to size a fan-out, batch it safely, and avoid the mapped tasks that overwhelm your scheduler and metadata database.", "url": "https://datalane-data.blog/blog/airflow-dynamic-task-mapping/", "published": "2026-05-19", "updated": "2026-05-19", "category": "Airflow", "tags": [ "airflow", "orchestration" ], "takeaways": [ "Every mapped index is a row in the metadata database and a scheduling decision. Four thousand of them is a scheduler problem, not a parallelism win.", "Batch the fan-out to the shape of the work. Fifty tasks of twenty files each beats a thousand tasks of one file.", "The list that drives expand is computed at run time by an upstream task, and if that task returns something huge it goes through XCom.", "max_active_tis_per_dag is the throttle that protects the downstream system. Pools protect everything else.", "A mapped task group gives you per-item multi-step pipelines without one DAG per item." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Iceberg vs Delta Lake in 2026: Metadata Design, Catalog Options, and How to Actually Choose", "summary": "The feature lists have converged. What still differs is metadata structure, catalog architecture, and engine support, and those are what decide which format fits your stack.", "url": "https://datalane-data.blog/blog/iceberg-vs-delta-lake/", "published": "2026-05-19", "updated": "2026-05-19", "category": "Iceberg", "tags": [ "iceberg", "delta-lake" ], "takeaways": [ "The feature comparison is over. Both formats have ACID commits, time travel, schema evolution, and row-level deletes. Stop choosing on features.", "The real difference is metadata topology: Iceberg uses an immutable snapshot tree with a catalog pointer swap, Delta uses a sequential log with periodic checkpoints.", "Iceberg hidden partitioning means queries do not need a partition predicate to prune. Delta needs the predicate, or liquid clustering instead of partitions.", "Catalog choice is a bigger commitment than format choice. The Iceberg REST protocol is the interoperability standard, and it is what makes a truly multi-engine lake possible.", "Choose by primary write engine, then expose the other format through UniForm or catalog federation rather than maintaining two copies of the data." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Testing Data Pipelines in Python: Fixtures, Warehouse Fakes, and Property-Based Transforms", "summary": "A layered testing strategy for Python data pipelines: pure transform tests, DuckDB as a warehouse fake, property-based invariants, and what belongs in CI versus production.", "url": "https://datalane-data.blog/blog/python-pipeline-testing-guide/", "published": "2026-05-18", "updated": "2026-05-18", "category": "Python", "tags": [ "python" ], "takeaways": [ "Untestable pipeline code is almost always a transform tangled with I/O. Separate them and the tests write themselves.", "Use DuckDB as a warehouse fake. It runs real SQL in-process and catches more than a mock ever will.", "Property-based tests find the empty frame, the null, and the duplicate key that your handcrafted fixture never included.", "Fixtures made from real data with real edge cases beat generated tidy data. Keep a small, ugly, committed sample.", "Unit tests prove the transform is correct. They cannot prove the data is. You need both, and they are different systems." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Slowly Changing Dimensions: Types 1 Through 6, and the Two You Will Actually Use", "summary": "Working SQL for every SCD type, the point-in-time join that most Type 2 implementations get wrong, and an honest account of which types are worth the maintenance.", "url": "https://datalane-data.blog/blog/slowly-changing-dimensions-guide/", "published": "2026-05-18", "updated": "2026-05-18", "category": "Data Modeling", "tags": [ "data-modeling", "sql" ], "takeaways": [ "You will ship Type 1 and Type 2. Types 3, 4, 5, and 6 are answers to questions almost nobody asks.", "Type 2 without a point-in-time join on the fact table is decoration. The history exists and nothing reads it.", "Choose per column, not per table. Most dimensions are Type 1 for typos and Type 2 for three attributes that matter.", "Late-arriving dimension rows are the failure mode that will actually cost you a weekend. Design for them on day one.", "A Type 2 dimension with no end-date test is a table that will silently develop overlapping versions." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Masking and Row Access Policies: Tag-Based Governance That Scales", "summary": "How to move from per-column masking policies to tag-based masking and mapping-table row access policies, including the performance traps and the audit queries that prove coverage.", "url": "https://datalane-data.blog/blog/snowflake-masking-and-row-access/", "published": "2026-05-18", "updated": "2026-05-18", "category": "Snowflake", "tags": [ "snowflake", "security", "governance" ], "takeaways": [ "Attaching masking policies column by column does not scale past about fifty columns. Attach them to tags instead and let the tag propagate.", "One policy per data type, not one per column. A masking policy signature is bound to a type, and that is the only dimension that needs to vary.", "Row access policies belong in a mapping table joined inside the policy body, never in a chain of hardcoded role names.", "A policy that references a mapping table adds a join to every query on that table. Keep the mapping small and clustered.", "Coverage is auditable from POLICY_REFERENCES and TAG_REFERENCES. If you cannot produce that list on demand, you do not have governance, you have policies." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Migrating from Synapse to Microsoft Fabric: What Maps Cleanly, What Does Not, and How to Sequence It", "summary": "A component-by-component migration map from Synapse to Fabric, the T-SQL and workload management gaps that bite, and the order I move things in to keep risk low.", "url": "https://datalane-data.blog/blog/synapse-to-fabric-migration/", "published": "2026-05-18", "updated": "2026-05-18", "category": "Azure", "tags": [ "azure", "fabric" ], "takeaways": [ "Spark pools and pipelines port almost unchanged. Dedicated SQL pools are the hard part and should move last.", "OneLake shortcuts mean you can adopt Fabric without copying a byte of ADLS Gen2 data. Start there.", "Fabric Warehouse is not the dedicated SQL pool T-SQL surface. Resource classes, distribution control, and several DDL features are gone.", "Capacity units with smoothing replace DWUs. The throttling behavior is different enough to invalidate your capacity math.", "Sequence by risk: Power BI and Direct Lake first, then Spark, then serverless SQL, then the warehouse." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Debezium on Postgres: Replication Slots, Snapshots, and Applying Changes Idempotently", "summary": "A production walkthrough of Postgres CDC with Debezium: how the replication slot fills your disk, how to snapshot a large table without locking it, and how to apply changes so replays are safe.", "url": "https://datalane-data.blog/blog/debezium-cdc-postgres-guide/", "published": "2026-05-17", "updated": "2026-05-17", "category": "Debezium", "tags": [ "debezium", "kafka", "cdc", "streaming" ], "takeaways": [ "A replication slot with no consumer will fill the primary database disk. This is the number one Debezium outage.", "One connector means one slot means one task. You scale Postgres CDC by splitting connectors, not by raising tasks.max.", "REPLICA IDENTITY FULL is what makes deletes and before-images usable, and it costs WAL volume.", "Incremental snapshots let you backfill a 500 GB table without stopping the stream or locking writes.", "CDC output is at-least-once and out of order across tables. Apply it with a MERGE keyed on the primary key and an LSN watermark." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airflow Datasets and Assets: Replacing Cron Guesswork With Data-Aware Scheduling", "summary": "How to move cross-team DAG dependencies from padded cron offsets to dataset-driven triggers, including conditional scheduling, naming rules, and the failure modes nobody warns you about.", "url": "https://datalane-data.blog/blog/airflow-datasets-data-aware-scheduling/", "published": "2026-05-16", "updated": "2026-05-16", "category": "Airflow", "tags": [ "airflow", "orchestration" ], "takeaways": [ "A cron offset between two DAGs is a guess about someone else runtime. Datasets replace the guess with an event.", "A dataset URI is a public interface. Once another team schedules on it, renaming it is a breaking change.", "A task that succeeds without writing anything still emits the update. Guard the outlet or your consumers run on stale data.", "Conditional scheduling with AND/OR is the feature that makes datasets usable across more than two teams.", "Datasets do not solve backfills. A backfill of a producer does not meaningfully replay downstream consumers." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Declarative Pipelines in Databricks: Expectations, Streaming Tables, and When the Abstraction Fights You", "summary": "What DLT genuinely buys you, how expectations and streaming tables behave in production, and the four situations where I still write plain Spark jobs instead.", "url": "https://datalane-data.blog/blog/databricks-dlt-pipelines/", "published": "2026-05-16", "updated": "2026-05-16", "category": "Databricks", "tags": [ "databricks", "pipelines" ], "takeaways": [ "DLT is not a transformation language, it is a dependency graph plus managed checkpoints, retries, and data quality gates. That plumbing is the actual product.", "Streaming tables are incremental and append-only from their source; materialized views recompute. Choosing wrong is the most expensive DLT mistake.", "Expectations with expect_or_drop quarantine bad rows without failing the pipeline, and the event log turns quality into a queryable time series.", "Full refresh is a loaded gun on streaming tables: it truncates and reprocesses everything, and it will do it to a table whose source has already aged out.", "I still use plain jobs for complex Python orchestration, external API side effects, and anything needing precise control over write timing." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Dataflow vs Dataproc: The Beam Model, the Cost Profiles, and Which One Your Team Can Actually Run", "summary": "A production comparison of Dataflow and Dataproc on GCP: what Beam buys you, how the two billing models differ, and the team characteristics that decide the choice.", "url": "https://datalane-data.blog/blog/dataflow-vs-dataproc/", "published": "2026-05-16", "updated": "2026-05-16", "category": "GCP", "tags": [ "gcp", "dataflow" ], "takeaways": [ "Dataflow is a runtime for one programming model. Dataproc is a cluster you own. That difference matters more than any feature comparison.", "Beam earns its complexity only if you run streaming with event-time windowing. For batch, it is a tax with no return.", "Dataproc on preemptible workers costs 40 to 70 percent less than equivalent Dataflow capacity, and you pay for that in operations.", "Dataflow Streaming Engine and Shuffle Service are separately billed and usually worth it. Turn them on and budget for them.", "The real deciding question is whether your team already writes Spark. Retraining a Spark team on Beam costs more than the infrastructure difference." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "ADF Pipeline Patterns That Scale: Parameterized Datasets, Metadata-Driven Ingestion, and Integration Runtime Sizing", "summary": "How to build one Data Factory pipeline that ingests 300 tables instead of 300 pipelines, and how to size integration runtimes so the bill and the SLA both work.", "url": "https://datalane-data.blog/blog/azure-data-factory-patterns/", "published": "2026-05-15", "updated": "2026-05-15", "category": "Azure", "tags": [ "azure", "adf" ], "takeaways": [ "One parameterized pipeline plus a control table beats 300 copied pipelines. The refactor pays for itself in a quarter.", "ADF charges per activity run, so a per-row ForEach is a pricing mistake before it is a performance one.", "Lookup returns at most 5,000 rows and 4 MB. Any metadata-driven design must page or partition around that.", "Data flow clusters take four to five minutes to start unless you set a TTL. Most ADF latency complaints are that.", "Self-hosted IR is a capacity decision, not a checkbox: size the VM, run two nodes, and watch the concurrent job count." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Pandas to Polars: What Translates, What Does Not, and What Breaks Quietly", "summary": "A production migration guide: the expression API, lazy frames and query optimization, null versus NaN semantics, and the pandas patterns with no polars equivalent.", "url": "https://datalane-data.blog/blog/python-pandas-to-polars-migration/", "published": "2026-05-15", "updated": "2026-05-15", "category": "Python", "tags": [ "python", "performance" ], "takeaways": [ "The speedup is real but it comes from the lazy engine, not the syntax. A line-by-line port of eager pandas leaves most of it on the table.", "Polars has no index. Half of a typical pandas codebase is index manipulation that simply deletes during migration.", "Null and NaN are different values in polars. Pandas conflated them, and that is where silent numeric differences come from.", "Any apply over rows is a wall. If you cannot express it as an expression, polars will not be faster than pandas.", "Migrate the heavy tail first. The 20% of jobs that consume 80% of memory pay for the effort; the rest can stay on pandas indefinitely." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Data Sharing: Secure Shares, Reader Accounts, and the Marketplace Without Shipping Copies", "summary": "How Snowflake secure data sharing works across accounts, regions, and clouds, when to use a reader account, and the governance patterns that keep a share from leaking more than you intended.", "url": "https://datalane-data.blog/blog/snowflake-data-sharing-guide/", "published": "2026-05-15", "updated": "2026-05-15", "category": "Snowflake", "tags": [ "snowflake", "governance", "architecture" ], "takeaways": [ "A share is metadata, not data. The consumer reads your micro-partitions directly and you keep paying storage while they pay compute.", "Never share a base table. Share a secure view, because a normal view leaks its definition and can leak rows through query plans.", "Cross-region and cross-cloud sharing requires replication, which turns a free share into a real storage and transfer bill.", "Reader accounts let you share with people who are not on Snowflake, but you pay their compute, so put a resource monitor on it before you send the URL.", "Row-level filtering by consumer account with CURRENT_ACCOUNT() is the pattern that makes one share serve many customers safely." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Star Schema vs One Big Table: What Actually Costs Money on a Columnar Warehouse", "summary": "Joins are not the expense they were in 2015. Here is what a wide denormalized table really costs in storage, rebuild time, and history, and the hybrid I ship instead of picking a side.", "url": "https://datalane-data.blog/blog/star-schema-vs-one-big-table/", "published": "2026-05-15", "updated": "2026-05-15", "category": "Data Modeling", "tags": [ "data-modeling", "sql" ], "takeaways": [ "A broadcast join against a dimension under a few million rows is close to free. Optimizing it away is solving a 2015 problem.", "One big table does not cost you storage. It costs you rebuild time and the ability to restate a dimension.", "Wide tables win on scan-heavy BI because column pruning means you only pay for what you select, not for the width.", "The moment a dimension attribute needs to change retroactively, one big table becomes a full-table rewrite. Price that before you commit.", "My default is a star in the gold layer plus purpose-built wide tables for specific dashboards, rebuilt from the star." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Airflow 3 for Pipeline Authors: What Actually Changed and What to Fix First", "summary": "The Airflow 3 changes that break real DAGs: the Task SDK boundary, logical_date becoming optional, removed context keys, DAG versioning, and a pre-upgrade checklist.", "url": "https://datalane-data.blog/blog/airflow-3-whats-new/", "published": "2026-05-14", "updated": "2026-05-14", "category": "Airflow", "tags": [ "airflow", "best-practices" ], "takeaways": [ "Task code can no longer touch the metadata database directly. Any operator that opens a session is a rewrite, not a config change.", "logical_date is now optional. Every DAG that formats a path from it will break on manual and asset-triggered runs.", "catchup defaults to False. That is a better default and a silent behavior change for DAGs that relied on the old one.", "DAG versioning means a run finishes with the code it started with. Mid-run deploys stop producing frankenruns.", "The migration is mostly mechanical. Budget your time for the custom operators and the context keys, not for the UI." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka Connect in Production: Converters, Dead Letter Queues, and Worker Sizing", "summary": "The Connect settings that decide whether your connectors survive contact with real data: converter choice, error tolerance with a DLQ, SMT limits, and how to size and scale workers.", "url": "https://datalane-data.blog/blog/kafka-connect-in-production/", "published": "2026-05-14", "updated": "2026-05-14", "category": "Kafka", "tags": [ "kafka", "streaming" ], "takeaways": [ "Converter mismatch is the most common Connect failure, and the error message never says \"converter\".", "Without errors.tolerance and a DLQ, one poison message stops a task and it stays stopped.", "Tasks, not workers, are the unit of parallelism. A source connector caps its own task count.", "SMTs are for reshaping single records. Anything needing a join or a lookup belongs in a stream processor.", "Connect config lives in git and is applied by CI. The REST API is for reading, not for Tuesday afternoon edits." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Cutting a Databricks Bill: Job Clusters, Spot, Photon, and Finding the Top Offenders in System Tables", "summary": "How I take 30 percent off a Databricks bill in a week: kill all-purpose clusters for jobs, price spot properly, test Photon per workload, and rank spend with system.billing.", "url": "https://datalane-data.blog/blog/databricks-job-cost-optimization/", "published": "2026-05-13", "updated": "2026-05-13", "category": "Databricks", "tags": [ "databricks", "cost" ], "takeaways": [ "All-purpose compute costs roughly 2 to 4 times job compute for the same DBU. Running scheduled work on it is the single most expensive habit in Databricks.", "system.billing.usage joined to list prices ranks your spend in one query. Every bill I have reviewed had three jobs accounting for over half of it.", "Photon is a DBU multiplier, not free speed. It wins on scans, joins, and aggregations; it loses on Python UDF pipelines where it cannot vectorize.", "Spot instances save 60 to 80 percent, but only with on-demand drivers, a fallback policy, and jobs that tolerate node loss. Never spot a driver.", "Autoscaling is not a cost strategy on short jobs. A fixed, correctly sized cluster beats an autoscaling one that spends half the run ramping up." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "DuckDB vs Spark on One Node: Where the Crossover Actually Happens", "summary": "Real timings and costs comparing single-node DuckDB against a Spark cluster, and why the crossover point is decided by shuffle volume rather than raw data size.", "url": "https://datalane-data.blog/blog/duckdb-vs-spark-single-node/", "published": "2026-05-13", "updated": "2026-05-13", "category": "DuckDB", "tags": [ "duckdb", "spark" ], "takeaways": [ "The crossover is not a data size. It is the point where your largest shuffle exceeds what one machine can spill efficiently.", "A 64 GB single node beat a 10-node Spark cluster on every scan-and-aggregate job I benchmarked, by 3 to 5x on wall clock and 20 to 100x on cost.", "Spark wins decisively on large shuffle-heavy joins, jobs needing more than about 400 GB of intermediate state, and anything that must survive node loss mid-run.", "Spark on a single node is the worst of both worlds: JVM overhead and shuffle machinery with no distribution benefit.", "Measure shuffle bytes in your Spark UI. If the biggest stage shuffles under 100 GB, the job probably belongs on one machine." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowpark vs SQL: When Python Wins and When It Is Just a Slower Way to Write SQL", "summary": "A production comparison of Snowpark Python and plain SQL in Snowflake, covering lazy DataFrames, UDF serialization costs, and the four workloads where Python genuinely earns its place.", "url": "https://datalane-data.blog/blog/snowpark-vs-sql/", "published": "2026-05-13", "updated": "2026-05-13", "category": "Snowflake", "tags": [ "snowflake", "python", "performance" ], "takeaways": [ "The Snowpark DataFrame API compiles to SQL and runs identically. Choosing it for a transform is a readability decision, not a performance one.", "The moment you write a Python UDF, you leave the vectorized engine and pay per-row serialization. That is where the real cost difference lives.", "Vectorized UDFs and UDTFs claw most of that back by processing pandas batches, but they still lose to native SQL for anything SQL can express.", "Snowpark earns its keep for model inference, complex procedural logic, existing Python libraries, and stored procedures that orchestrate work.", "Always print the generated SQL before you defend a Snowpark pipeline. Half the performance complaints I have seen were a hidden collect() in a loop." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Six Ways to Deduplicate a Table: What Each Costs and Which Row Survives", "summary": "DISTINCT, GROUP BY, ROW_NUMBER, QUALIFY, MERGE, and stream-side dedup compared on cost and semantics, plus how to pick the survivor row deterministically.", "url": "https://datalane-data.blog/blog/sql-deduplication-strategies/", "published": "2026-05-13", "updated": "2026-05-13", "category": "SQL", "tags": [ "sql", "performance" ], "takeaways": [ "Deduplication has two separate questions: which rows are duplicates, and which duplicate survives. Most bugs come from answering only the first.", "DISTINCT only works when duplicates are identical across every column. One differing timestamp and it does nothing.", "ROW_NUMBER without a unique tiebreaker is non-deterministic. The same input produces different output across runs.", "Deduplicating at read time forever is more expensive than fixing the write path once.", "Late-arriving duplicates in incremental models need a merge key and a lookback window, not a bigger DISTINCT." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Performance at Scale: Threads, Materializations, and Per-Model Cost Attribution", "summary": "Cutting dbt run time and warehouse spend on a large project: how to find the real critical path, when threads stop helping, and how to attribute cost to individual models.", "url": "https://datalane-data.blog/blog/dbt-performance-on-large-projects/", "published": "2026-05-12", "updated": "2026-05-12", "category": "dbt", "tags": [ "dbt", "cost-optimization" ], "takeaways": [ "Total run time is set by the critical path, not the sum of model times. Optimizing a slow model off the path buys you nothing.", "Threads past the point where your warehouse queues are just a longer queue with more concurrency contention.", "run_results.json plus your warehouse query history gives you per-model cost. Nobody does this and everybody should.", "The cheapest optimization on most projects is deleting models nobody queries. Second cheapest is turning a table into a view.", "Incremental is not automatically faster. A merge on an unclustered billion-row table can cost more than a full rebuild." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Batch LLM Enrichment: Idempotency, Cost Per Row, and Knowing When the Output Is Wrong", "summary": "Treating an LLM call as a transformation step in a batch pipeline. Content-hash idempotency, per-row cost budgets, retry semantics for a non-deterministic API, and how to test output you cannot diff.", "url": "https://datalane-data.blog/blog/llm-data-pipeline-patterns/", "published": "2026-05-12", "updated": "2026-05-12", "category": "AI & GenAI", "tags": [ "ai", "python" ], "takeaways": [ "Key every enrichment on a content hash plus a prompt version. Re-running should cost zero and change nothing.", "Cost per row is a design constraint you set before writing the prompt, not a number you discover in the invoice.", "A malformed JSON response is a retry. A confidently wrong classification is a data quality incident, and only one of them raises an exception.", "Batch APIs are half price and you almost always have the latency budget. Use them by default.", "You cannot diff LLM output, so pin a labeled sample and score every prompt change against it before it reaches the mart." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Step Functions vs Airflow for Data Orchestration: Cost, Observability, and the Point Where You Outgrow One", "summary": "State transitions versus scheduler hours, execution history versus task logs, and the specific signals that tell you which orchestrator your pipeline has outgrown.", "url": "https://datalane-data.blog/blog/step-functions-vs-airflow/", "published": "2026-05-12", "updated": "2026-05-12", "category": "AWS", "tags": [ "aws", "orchestration" ], "takeaways": [ "Step Functions orchestrates AWS services reliably. Airflow orchestrates data with dependencies, backfills, and lineage.", "Standard workflows cost about $0.025 per 1,000 state transitions. A loop over 50,000 files is a real invoice.", "MWAA has a floor of roughly $350 to $400 a month before you run a single task. Step Functions has no floor.", "The moment you need a backfill across a date range with dependency-aware reruns, Step Functions stops being the answer.", "Running both is normal: Airflow owns the schedule and the data graph, Step Functions owns the event-driven AWS choreography." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Time Travel and Cloning: What They Really Cost and How to Migrate Safely", "summary": "How Time Travel retention, fail-safe, and zero-copy clones bill against storage, plus the migration and recovery patterns that make the storage cost worth paying.", "url": "https://datalane-data.blog/blog/snowflake-time-travel-and-cloning/", "published": "2026-05-11", "updated": "2026-05-11", "category": "Snowflake", "tags": [ "snowflake", "cost-optimization", "governance" ], "takeaways": [ "Time Travel storage scales with churn, not table size. A 200 GB table rewritten nightly can carry terabytes of retention.", "Fail-safe is seven extra days you cannot query, cannot disable, and still pay for on every permanent table.", "Transient tables drop fail-safe entirely, which is the right default for staging, scratch, and anything reproducible.", "A zero-copy clone is free until you write to it. Cloning prod to dev then rebuilding everything is not free at all.", "Clone before every risky migration. It is the cheapest rollback plan Snowflake offers and it takes one statement." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Structured Streaming in Production: Triggers, Checkpoints, Watermarks, and Exactly-Once Sinks", "summary": "The four things that decide whether a Spark stream survives a year in production: trigger choice, checkpoint discipline, watermark math, and what exactly-once really guarantees.", "url": "https://datalane-data.blog/blog/spark-structured-streaming-guide/", "published": "2026-05-11", "updated": "2026-05-11", "category": "Apache Spark", "tags": [ "spark", "streaming" ], "takeaways": [ "The checkpoint directory is the stream. Delete it and you have a new stream with no memory of what it read; change the query too much and it will refuse to start from it.", "Most teams do not need continuous streaming. AvailableNow triggers on a 15-minute schedule cost a fraction of an always-on cluster and are easier to operate.", "A watermark is a promise about lateness, and Spark enforces it by dropping data. Pick it from measured event-time lag, not from a round number.", "Exactly-once is a property of the source, the checkpoint, and the sink together. An idempotent sink with offset tracking gets you there; a REST call in foreachBatch does not.", "A one-minute trigger writing to 24 partitions creates up to 34,000 files a day. Compaction is not optional, it is part of the stream design." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Recursive CTEs in Production: Org Charts, Bill of Materials, and Cycle Protection", "summary": "How WITH RECURSIVE actually executes, patterns for hierarchy traversal and BOM explosion, and the cycle guards that keep a bad edge from running forever.", "url": "https://datalane-data.blog/blog/sql-recursive-cte-hierarchies/", "published": "2026-05-11", "updated": "2026-05-11", "category": "SQL", "tags": [ "sql" ], "takeaways": [ "A recursive CTE is iteration, not recursion. Each pass reads only the previous pass output, which is why it terminates.", "Every recursive query in production needs a depth cap. One self-referencing row turns a report into an unbounded loop.", "Carry the path as an array or string and filter on it. That is the only cycle guard that works when the cycle is longer than one hop.", "Bill of materials explosion multiplies quantities down the path. Forget the multiplication and your material requirements are wrong, not missing.", "For a hierarchy read thousands of times per day, materialize a closure table nightly instead of recursing per query." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Where Lambda Belongs in a Data Pipeline and Where It Quietly Becomes a Distributed System You Cannot Debug", "summary": "Lambda is excellent glue and a terrible framework. Here are the limits that shape every design, the idempotency you must build yourself, and the point to stop.", "url": "https://datalane-data.blog/blog/aws-lambda-for-data-pipelines/", "published": "2026-05-10", "updated": "2026-05-10", "category": "AWS", "tags": [ "aws", "lambda" ], "takeaways": [ "Lambda is the best event router in AWS and the worst place to put a multi-step workflow.", "The 15-minute timeout is not the real limit. Retry semantics and concurrency are what break pipelines.", "Every Lambda in a data path needs an idempotency key backed by a conditional write, or you will double-count.", "Concurrency is an account-level shared resource. One runaway function throttles every other function you own.", "When you are writing state machines in Lambda code, you have built Step Functions badly. Stop and use the real one." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "DuckDB in Production: The Jobs It Genuinely Wins and the Single-Node Limits You Will Hit", "summary": "Where DuckDB belongs in a real pipeline, how far out-of-core execution gets you on one machine, and the concurrency and memory walls that decide when to stop.", "url": "https://datalane-data.blog/blog/duckdb-for-production-pipelines/", "published": "2026-05-10", "updated": "2026-05-10", "category": "DuckDB", "tags": [ "duckdb", "python" ], "takeaways": [ "A 16 GB container running DuckDB handles most jobs that teams currently pay a Spark cluster to run.", "Out-of-core execution means DuckDB survives datasets larger than memory, but joins and sorts still need roughly 2x the working-set size on disk.", "One writer per database file, full stop. If you need concurrent writers, DuckDB is the wrong component.", "Push predicates and column lists into read_parquet so DuckDB skips files. It will not save you from a full scan you asked for.", "The right production shape is a stateless container that reads object storage, writes object storage, and keeps no long-lived database file." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka Schema Registry: Evolving Event Schemas Without Breaking Consumers", "summary": "Avro versus Protobuf, what each compatibility mode really allows, and the CI gate that stops a breaking schema change before it reaches a topic your consumers cannot read.", "url": "https://datalane-data.blog/blog/kafka-schema-registry-evolution/", "published": "2026-05-10", "updated": "2026-05-10", "category": "Kafka", "tags": [ "kafka", "streaming" ], "takeaways": [ "BACKWARD compatibility is the right default, and it means new consumers read old data, not the reverse.", "A field without a default is a breaking change in Avro, no matter how optional it feels to the author.", "Protobuf gives you field numbers and cheap evolution. Avro gives you a self-describing schema the lake already understands.", "Register schemas in CI against the live registry. Discovering incompatibility at produce time is too late.", "FULL_TRANSITIVE sounds safest and will freeze your schema. Pick it deliberately, not defensively." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Macros and Jinja: The Patterns That Earn Their Keep, and the Clever Ones That Do Not", "summary": "Which Jinja patterns are worth the indirection in a production dbt project, how to test macros, and the metaprogramming that quietly makes a project unmaintainable.", "url": "https://datalane-data.blog/blog/dbt-macros-and-jinja-patterns/", "published": "2026-05-09", "updated": "2026-05-09", "category": "dbt", "tags": [ "dbt", "best-practices" ], "takeaways": [ "A macro is worth writing when the same SQL appears three times and would change in all three places at once. Two is a coincidence.", "Every macro adds a compile step between what you wrote and what ran. Price that indirection before you pay it.", "run_query at parse time hits your warehouse on every dbt command, including dbt ls and dbt parse. It is the single most expensive habit in Jinja.", "Macros that generate model SQL from a config dict turn your DAG into a program nobody can read a diff of.", "The best macro in most projects is a twelve-line one that centralizes a business definition, not a framework." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Pivot and Unpivot in SQL: Patterns That Survive a Changing Column Set", "summary": "Conditional aggregation, native PIVOT syntax, unpivoting wide tables, and how to handle dynamic categories without generating SQL at runtime.", "url": "https://datalane-data.blog/blog/sql-pivot-unpivot-patterns/", "published": "2026-05-09", "updated": "2026-05-09", "category": "SQL", "tags": [ "sql" ], "takeaways": [ "Conditional aggregation is the portable pivot. Native PIVOT syntax is sugar over the same plan, not a faster one.", "A pivot whose column list comes from data is a schema that changes without a migration. Push it to the BI layer instead.", "Unpivoting a wide table into key-value pairs is the fix for 90% of dynamic pivot requests, and it needs no dynamic SQL.", "MAX with a CASE is the right pivot for a single value per cell. SUM silently gives you zero instead of NULL for missing cells.", "Pivots hide grain bugs: if a cell has two source rows, aggregation swallows one and nobody sees an error." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Vector Search in the Warehouse: When You Do Not Need a Vector Database", "summary": "Native vector search in Snowflake, BigQuery, and pgvector has quietly become good enough for most RAG workloads. Here are the thresholds, the real costs, and when a dedicated store still wins.", "url": "https://datalane-data.blog/blog/vector-search-in-the-warehouse/", "published": "2026-05-09", "updated": "2026-05-09", "category": "AI & GenAI", "tags": [ "ai", "rag" ], "takeaways": [ "Under roughly 5 million vectors with batch-refreshed content, the warehouse wins on total cost of ownership every time.", "The sync pipeline you avoid is worth more than the milliseconds you lose. Two copies of embeddings drift the week you stop watching.", "pgvector with HNSW handles a surprising amount of production traffic. I have run 3 million vectors at p95 under 40ms on one instance.", "Metadata filtering is where dedicated stores still hurt, and where SQL is unfairly good.", "Move out when you need sub-20ms p99 on a live user path, not when a benchmark chart says so." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Redshift Serverless vs Snowflake in 2026: Architecture, Concurrency, Cost Model, and Migration Friction", "summary": "A production comparison of Redshift Serverless and Snowflake: how RPU-seconds differ from credits, where each one bends under concurrency, and what actually breaks in a migration.", "url": "https://datalane-data.blog/blog/redshift-vs-snowflake/", "published": "2026-05-08", "updated": "2026-05-08", "category": "AWS", "tags": [ "aws", "warehouse" ], "takeaways": [ "Both separate storage from compute now. The architecture argument is over; the operational and cost arguments are not.", "Redshift Serverless bills RPU-seconds with a 60-second minimum and scales one workgroup. Snowflake bills credits per warehouse and lets you run many.", "Snowflake wins on concurrency isolation because a warehouse per workload is a five-second change, not a capacity negotiation.", "Redshift wins on price when your workload is a predictable ETL window inside an AWS account you already pay for.", "Migration friction is 20 percent SQL dialect and 80 percent distribution keys, procedures, and the tooling wired around them." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake RBAC Design: A Role Hierarchy That Survives an Audit and a Reorg", "summary": "How to split functional roles from access roles, use future grants and managed access schemas, and build a Snowflake permission model that stays reviewable as teams change.", "url": "https://datalane-data.blog/blog/snowflake-rbac-design/", "published": "2026-05-08", "updated": "2026-05-08", "category": "Snowflake", "tags": [ "snowflake", "security", "governance" ], "takeaways": [ "Split roles into access roles that hold privileges and functional roles that hold people. Never grant an object privilege to a person-facing role.", "Future grants are what keep the model correct on Monday. Without them, every new table is a silent permission gap.", "Managed access schemas stop object owners from granting privileges behind your back, which is what auditors actually check.", "Ownership belongs to a role, never a user. Any grant traceable to a departed employee is an audit finding.", "If your RBAC model is not in version control and applied by CI, it is not a model, it is a folklore." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Spark Executor Memory: Spill, GC, and the OOM Errors That Are Really Partition-Sizing Errors", "summary": "How Spark executor memory is actually divided, why most OOMs are one oversized partition rather than a small cluster, and the six failures I can now diagnose from the error text alone.", "url": "https://datalane-data.blog/blog/spark-memory-management-oom/", "published": "2026-05-08", "updated": "2026-05-08", "category": "Apache Spark", "tags": [ "spark", "performance" ], "takeaways": [ "Executor memory is four regions, and only the unified pool grows and shrinks. Knowing which region ran out tells you which fix to apply.", "Almost every executor OOM I have debugged was a partition-sizing problem. Raising memory hides it for one release; sizing partitions fixes it.", "Spill to disk is not a failure, it is a warning. Spill larger than shuffle write means every byte moves at least three times.", "A container killed by YARN or Kubernetes for exceeding limits is an overhead problem, not a heap problem, and raising executor memory makes it worse.", "Driver OOM is nearly always collect, toPandas, or an oversized broadcast. It is a code review finding, not a capacity finding." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "How BigQuery Pruning Actually Works: Partition Metadata, Block Statistics, and the Shapes That Defeat Both", "summary": "What BigQuery reads before it reads your data: partition metadata, Capacitor block statistics, and the specific query shapes that silently turn a pruned scan into a full one.", "url": "https://datalane-data.blog/blog/bigquery-pruning-internals/", "published": "2026-05-07", "updated": "2026-05-07", "category": "BigQuery", "tags": [ "bigquery", "performance" ], "takeaways": [ "Partition pruning happens at planning time from metadata. Block pruning happens at execution time, which is why the pre-run estimate is always pessimistic.", "A query parameter or a subquery in the partition predicate defers pruning to runtime. You still get pruned, but the estimate and maximum_bytes_billed see the full table.", "Clustering prunes by min/max per block, so a low-cardinality first cluster column and a sorted second column behave completely differently.", "Join-key pruning only fires when BigQuery can build a runtime filter from the small side. An unfiltered dimension prunes nothing on the fact.", "Verify pruning with total_bytes_processed from JOBS_BY_PROJECT, not the editor estimate. They disagree by design." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Snapshots for SCD Type 2: Check vs Timestamp, and When to Hand-Roll", "summary": "Choosing between check and timestamp snapshot strategies, what happens when a snapshot run is missed, and the cases where a hand-rolled SCD2 model beats dbt snapshot.", "url": "https://datalane-data.blog/blog/dbt-snapshots-scd2/", "published": "2026-05-07", "updated": "2026-05-07", "category": "dbt", "tags": [ "dbt", "sql" ], "takeaways": [ "A snapshot only knows what it saw. Miss a run, and the change that happened in between is gone forever.", "Timestamp strategy is cheaper and more honest, but only if the source updated_at is actually updated by the source.", "Check strategy on all columns turns every schema addition into a full re-versioning of your history table.", "Snapshot the staging model, not the raw source, so casting and soft-delete handling happen before history is frozen.", "If the source already ships change events with timestamps, hand-roll the SCD2 build. dbt snapshot is for sources that only expose current state." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka Exactly-Once Semantics: What Transactions Actually Guarantee", "summary": "Idempotent producers, transactions, and read_committed give you exactly-once inside Kafka. The moment you write to a database, you still need an idempotent sink. Here is where the boundary sits.", "url": "https://datalane-data.blog/blog/kafka-exactly-once-semantics/", "published": "2026-05-07", "updated": "2026-05-07", "category": "Kafka", "tags": [ "kafka", "streaming" ], "takeaways": [ "Exactly-once in Kafka means exactly-once from Kafka to Kafka. It says nothing about your database.", "Turn on the idempotent producer everywhere. It is free, on by default in modern clients, and kills the retry duplicate.", "Transactions only help when the consume, process, and produce all live inside Kafka.", "Every external sink needs its own idempotency key. A MERGE on a natural key beats a transaction you cannot join.", "read_committed costs you latency equal to the slowest open transaction, not the average one." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Warehouse Sizing: The Spill and Queue Signals That Tell You Up or Out", "summary": "How to pick a Snowflake warehouse size from evidence instead of intuition, using spilling, queueing, and scaling tests to decide between a bigger warehouse and more clusters.", "url": "https://datalane-data.blog/blog/snowflake-warehouse-sizing-guide/", "published": "2026-05-06", "updated": "2026-05-06", "category": "Snowflake", "tags": [ "snowflake", "performance", "cost-optimization" ], "takeaways": [ "Doubling warehouse size doubles the credit rate, so a query that scales linearly costs exactly the same and finishes twice as fast.", "Remote spilling is the one unambiguous signal to size up. Local spilling is a warning, not an emergency.", "Queueing means add clusters, not size. Mixing the two fixes is how teams end up with an idle XL.", "Test a size change with the result cache off and the same data, or you are measuring caching, not compute.", "Start every new warehouse at X-Small and let evidence promote it. Nobody has ever regretted starting too small." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Spark Data Skew: Diagnosing It Properly, Then Fixing It With Salting, AQE, and Broadcast", "summary": "One task at 40 minutes while 399 finished in 20 seconds is skew. Here is how I find the offending key, and the four fixes ranked by how much they cost me.", "url": "https://datalane-data.blog/blog/spark-data-skew-fixes/", "published": "2026-05-06", "updated": "2026-05-06", "category": "Apache Spark", "tags": [ "spark", "performance" ], "takeaways": [ "Skew is diagnosed from the task duration distribution, not from job runtime. Max divided by median above 5 is the threshold I act on.", "Most skew in real data is one of three things: a NULL key, a synthetic default like -1 or unknown, or a genuine whale customer.", "AQE skew join handles moderate skew for free, but it only splits partitions above both a ratio and an absolute size threshold. A single monster key still lands in one task.", "Salting works and is ugly. Only salt the hot keys, keep the tail unsalted, and union the two paths.", "Filtering NULL join keys before the join is the cheapest skew fix that exists, and it fixes a correctness smell at the same time." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "CTE vs Subquery: When the WITH Clause Is a Materialization Barrier", "summary": "What actually happens to a CTE in Postgres, Snowflake, and BigQuery, when it blocks predicate pushdown, and when referencing it twice costs you two scans.", "url": "https://datalane-data.blog/blog/sql-cte-vs-subquery-performance/", "published": "2026-05-06", "updated": "2026-05-06", "category": "SQL", "tags": [ "sql", "performance" ], "takeaways": [ "A CTE is not a temp table on modern engines. It is usually inlined, which means referencing it twice runs it twice.", "Postgres inlined CTEs starting in version 12. Before that, every WITH was an optimization fence and rewrites were mandatory.", "The real performance question is never CTE versus subquery. It is inlined versus materialized, and only one engine lets you say which.", "A CTE referenced three times in a Snowflake model is three scans unless you make it a temp table or a dbt ephemeral boundary.", "Readability wins by default. Reach for materialization only when the profile shows the same scan repeated." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Text-to-SQL in Production: Why the Demo Works and the Deployment Fails", "summary": "The failure modes that only show up after launch — join paths, metric drift, runaway cost — and the semantic layer, constrained generation, and eval harness that make text-to-SQL survivable.", "url": "https://datalane-data.blog/blog/text-to-sql-in-production/", "published": "2026-05-06", "updated": "2026-05-06", "category": "AI & GenAI", "tags": [ "ai", "sql" ], "takeaways": [ "The demo works because the demo schema has five tables and one meaning per column. Production has neither.", "Stop generating SQL against tables. Generate a query spec against a semantic layer and compile the SQL yourself.", "Execution accuracy on a golden set is the only launch metric that matters. Syntactic validity is table stakes.", "Budget for cost per question, not cost per token. One unpruned scan can outspend a month of model calls.", "Always show the generated SQL. A hidden query turns a wrong number into a trusted wrong number." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Making Athena Fast and Cheap: Partition Projection, Parquet Layout, CTAS, and Workgroup Byte Limits", "summary": "Athena bills you for bytes scanned. Here is the partition projection, Parquet layout, CTAS compaction, and workgroup guardrails I use to cut that number by 90 percent.", "url": "https://datalane-data.blog/blog/athena-cost-and-performance/", "published": "2026-05-05", "updated": "2026-05-05", "category": "AWS", "tags": [ "aws", "cost" ], "takeaways": [ "Athena has exactly one cost lever: bytes scanned. Every optimization is a way to read fewer bytes.", "Partition projection removes the Glue metastore round trip and is the single biggest latency win on wide tables.", "Target 128 MB to 512 MB Parquet files. Thousands of small files cost more in listing time than in scan bytes.", "CTAS with bucketing and a sort order is your compaction job. You do not need Spark for most rewrites.", "Set a per-query bytes-scanned limit on every workgroup before someone runs SELECT * on the raw zone." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Cutting a BigQuery Bill: Partitioning, Byte Limits, Materialized Views, and On-Demand vs Editions", "summary": "How I take a runaway BigQuery invoice apart: find the top jobs, fix pruning, cap bytes billed, use materialized views honestly, and decide when Editions beats on-demand.", "url": "https://datalane-data.blog/blog/bigquery-cost-optimization/", "published": "2026-05-05", "updated": "2026-05-05", "category": "BigQuery", "tags": [ "bigquery", "cost" ], "takeaways": [ "Ninety percent of a bloated BigQuery bill comes from under twenty queries. Read JOBS_BY_PROJECT before you change a table.", "maximum_bytes_billed is the only control that turns a cost incident into a failed query. Set it at the project level.", "Materialized views only pay when the base table changes far less often than the view is read.", "On-demand beats Editions until you are spending steadily above roughly the equivalent of 100 baseline slots.", "Physical storage billing cuts the storage line 40 to 60 percent on well-compressed tables and takes one ALTER statement." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "dbt Project Structure: A Layout That Survives Two Years and Forty Models", "summary": "How I lay out staging, intermediate, and marts so a dbt project still makes sense after two years: naming rules, ref boundaries, and the folders that always rot.", "url": "https://datalane-data.blog/blog/dbt-project-structure-guide/", "published": "2026-05-05", "updated": "2026-05-05", "category": "dbt", "tags": [ "dbt", "best-practices" ], "takeaways": [ "Layers are a dependency rule, not a folder convention. If staging can ref a mart, you have one layer with three folder names.", "One staging model per source table, forever. The moment two sources join in staging, the layer stops being a translation layer.", "Intermediate models are private. Nothing outside their own mart lineage should ref them, and the folder name should say so.", "Name models after their grain, not their source system. fct_orders survives a Fivetran migration; shopify_orders_final does not.", "A model with more than about a dozen upstream refs is not a model, it is a missing layer." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kafka Consumer Group Rebalancing: Why Your Consumers Stall and How to Stop It", "summary": "Rebalances are the most common cause of Kafka consumer lag spikes. Here is the protocol, the four timeouts that matter, and the cooperative sticky config I ship by default.", "url": "https://datalane-data.blog/blog/kafka-consumer-group-rebalancing/", "published": "2026-05-05", "updated": "2026-05-05", "category": "Kafka", "tags": [ "kafka", "streaming" ], "takeaways": [ "Most rebalance storms are not failures. They are a slow poll loop tripping max.poll.interval.ms.", "Eager assignment stops the whole group. Cooperative sticky only moves the partitions that must move.", "session.timeout.ms detects dead consumers. max.poll.interval.ms detects slow ones. They are different problems.", "Static group membership turns a rolling restart from N rebalances into zero.", "If you cannot answer \"how long does one poll take at p99\", you cannot tune any of these numbers." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "The Snowflake Cost Optimization Playbook: Cut the Bill Without Anyone Noticing", "summary": "A ranked playbook for reducing Snowflake spend: auto-suspend defaults, right-sizing warehouses, resource monitors that actually fire, and the ACCOUNT_USAGE queries that name your top ten offenders.", "url": "https://datalane-data.blog/blog/snowflake-cost-optimization-playbook/", "published": "2026-05-04", "updated": "2026-05-04", "category": "Snowflake", "tags": [ "snowflake", "cost-optimization" ], "takeaways": [ "Most Snowflake overspend is idle warehouse time and a handful of repeat-offender queries, not expensive storage.", "Auto-suspend at 60 seconds is the right default for everything except a warehouse serving continuous BI.", "Ten queries usually own a third of the bill. Find them in QUERY_HISTORY before you touch a single warehouse size.", "Resource monitors should notify long before they suspend, and every warehouse needs one even if it never fires.", "Attribution beats austerity: a tagged warehouse with a named owner gets optimized, an anonymous one grows forever." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Spark Shuffle Explained: What It Costs, How to Read It in the UI, and the Rewrites That Remove It", "summary": "A shuffle is a disk write, a network fetch, and a sort you did not ask for. Here is what it costs, how to spot it in the Spark UI, and the query rewrites that delete it.", "url": "https://datalane-data.blog/blog/spark-shuffle-explained/", "published": "2026-05-04", "updated": "2026-05-04", "category": "Apache Spark", "tags": [ "spark", "performance" ], "takeaways": [ "A shuffle writes every row to local disk, then pulls it across the network. It is the only part of Spark that touches all three slow resources at once.", "Stage boundaries in the Spark UI are shuffles. If your DAG has six stages, you paid for five shuffles, whether you meant to or not.", "Shuffle Spill (Disk) larger than Shuffle Write means your partitions do not fit in execution memory, and no amount of extra executors fixes that.", "The cheapest shuffle is the one you delete: broadcast the small side, aggregate before joining, and stop calling repartition out of superstition.", "AQE coalesces partitions after the fact, but it cannot undo a shuffle you never needed. Fix the plan first, then tune." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Window Frames Deep Dive: RANGE vs ROWS and the Patterns That Replace Self-Joins", "summary": "How window frames actually evaluate, why RANGE and ROWS give different answers on ties, and six window patterns that delete self-joins from your models.", "url": "https://datalane-data.blog/blog/sql-window-functions-deep-dive/", "published": "2026-05-04", "updated": "2026-05-04", "category": "SQL", "tags": [ "sql", "performance" ], "takeaways": [ "The default frame is RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, and it silently sums all tied rows at once.", "If your ORDER BY column has duplicates, ROWS and RANGE return different running totals. One of them is the bug.", "Any self-join that compares a row to its neighbors is a window function you have not written yet.", "Every extra PARTITION BY or ORDER BY spec adds a sort. Reuse one WINDOW clause and the engine sorts once.", "Filtering on a window result needs QUALIFY or a subquery, because WHERE runs before the window is computed." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Cortex Analyst and Cortex Search: Shipping Chat With Your Data That Answers Correctly", "summary": "How Cortex Analyst semantic models and Cortex Search hybrid retrieval fit together, what the YAML actually controls, and where the accuracy comes from.", "url": "https://datalane-data.blog/blog/snowflake-cortex-analyst-search/", "published": "2026-04-26", "updated": "2026-04-26", "category": "AI & GenAI", "tags": [ "ai", "snowflake", "cortex" ], "takeaways": [ "Cortex Analyst answers metric questions by compiling against your semantic model YAML. The YAML quality is the accuracy ceiling.", "Cortex Search is hybrid retrieval - vector plus keyword - as a managed service with its own refresh lag and warehouse bill.", "Verified queries in the semantic model are the highest-leverage accuracy feature. Add one per recurring question.", "Route between Analyst and Search in your app layer. Neither service routes for you.", "Both run inside the Snowflake perimeter under a role you define, which is the actual reason to pick them over external stacks." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Kinesis Data Streams vs Firehose: Shards, On-Demand, and Landing Streams in S3", "summary": "When Kinesis Data Streams earns its shards, when Firehose is enough, and the buffering and partitioning patterns that keep S3 landings queryable.", "url": "https://datalane-data.blog/blog/kinesis-streams-firehose-patterns/", "published": "2026-04-25", "updated": "2026-04-25", "category": "AWS", "tags": [ "aws", "streaming" ], "takeaways": [ "Firehose is a delivery pipe, not a stream. If no application reads records in flight, you probably want Firehose alone.", "A shard is 1 MB/s or 1000 records/s in, 2 MB/s out. Hot partition keys break these limits long before totals do.", "On-demand mode removes shard math but not partition-key discipline, and it costs more at steady sustained throughput.", "Firehose buffering is a freshness contract: 60 seconds to 900 seconds, or 1 to 128 MB, whichever comes first.", "Dynamic partitioning in Firehose writes Hive-style S3 prefixes so Athena and Spark can prune without a compaction job first." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Joins and the Fan-Out Bug: How Revenue Doubles Without an Error", "summary": "The join fan-out failure class: grain checks before and after every join, EXISTS for semi-joins, anti-joins done right, and the postmortem pattern.", "url": "https://datalane-data.blog/blog/sql-joins-explained-fan-out/", "published": "2026-04-24", "updated": "2026-04-24", "category": "SQL", "tags": [ "sql" ], "takeaways": [ "Fan-out happens when the join key is not unique on the side you assumed. Revenue doubles and nothing errors.", "Every join has an expected grain. Assert it with a row-count or key-uniqueness check, not with hope.", "Use EXISTS when you are filtering, not joining. A semi-join cannot fan out by construction.", "Anti-joins with NOT IN break on NULLs. Use NOT EXISTS or a LEFT JOIN with an IS NULL guard.", "The postmortem is always the same: a dimension quietly gained duplicates and no test was watching its key." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Slim CI in dbt: state:modified+ and --defer on a 1,000-Model Project", "summary": "How to get dbt CI from 50 minutes to 4 with state comparison and defer: manifest artifacts, GitHub Actions wiring, and the edge cases that bite.", "url": "https://datalane-data.blog/blog/dbt-slim-ci-state-defer/", "published": "2026-04-23", "updated": "2026-04-23", "category": "dbt", "tags": [ "dbt", "ci-cd" ], "takeaways": [ "Slim CI builds only what a PR changed plus its children, and borrows everything else from prod with --defer.", "state:modified compares your PR manifest against a stored prod manifest. Where that artifact comes from decides whether CI is trustworthy.", "A one-line macro edit marks every model that calls it as modified. That is correct, and you need a plan for it.", "Environment-dependent configs like different schemas per target create false positives. state_modified_compare_more_unrendered_values fixes most of them.", "Always keep a fallback: if the prod manifest is missing, CI should build a deliberate subset, not silently build nothing." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Unity Catalog in Practice: Metastore Layout, Grants That Scale, and Leaving hive_metastore", "summary": "How I structure Unity Catalog for real teams: three-level namespace design, group-based grants, lineage that works, and a hive_metastore migration that ships.", "url": "https://datalane-data.blog/blog/databricks-unity-catalog-guide/", "published": "2026-04-22", "updated": "2026-04-22", "category": "Databricks", "tags": [ "databricks", "governance" ], "takeaways": [ "One metastore per region, catalogs per environment or domain, schemas per layer. Decide this before table one, not after table four hundred.", "Grant to groups synced from your identity provider, never to individual users. USE CATALOG and USE SCHEMA are the gates everyone forgets.", "Managed tables under Unity Catalog get you predictive optimization and safe defaults. External tables are for data other engines must own.", "Lineage is automatic and column-level, but only for workloads that run through UC-enabled compute. Legacy clusters are invisible.", "Migrate from hive_metastore table by table with SYNC or CTAS, repoint jobs, then lock the old namespace read-only. Big-bang migrations stall." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Search Optimization Service: Point Lookups Without the Full Scan", "summary": "How the search access path works, what it costs to build and maintain, which predicates it serves, and when clustering keys are the better buy.", "url": "https://datalane-data.blog/blog/snowflake-search-optimization-service/", "published": "2026-04-21", "updated": "2026-04-21", "category": "Snowflake", "tags": [ "snowflake", "performance" ], "takeaways": [ "Search Optimization builds a persistent access path that maps values to micro-partitions, so equality lookups skip the scan.", "It serves high-cardinality point lookups where clustering is worst: UUIDs, emails, order numbers.", "Scope it to columns with ON EQUALITY. Table-wide enablement is how the storage and maintenance bill doubles for nothing.", "Costs land in three places: build credits, ongoing maintenance proportional to churn, and extra storage roughly 10-25% of the table.", "ESTIMATE_SEARCH_OPTIMIZATION_COSTS gives you the price before you commit. Run it every time." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Snowflake Clustering Keys: When They Pay for Themselves and When They Burn Credits", "summary": "Clustering depth, what automatic reclustering actually costs, and a decision framework for clustering keys vs Search Optimization vs doing nothing.", "url": "https://datalane-data.blog/blog/snowflake-clustering-keys-guide/", "published": "2026-04-14", "updated": "2026-04-14", "category": "Snowflake", "tags": [ "snowflake", "performance" ], "takeaways": [ "A clustering key is a standing order to keep rewriting the table. You pay for it on every churn, forever.", "Most tables under a terabyte do not need one. Natural ingestion order already clusters by load time.", "SYSTEM$CLUSTERING_INFORMATION tells you depth today. AUTOMATIC_CLUSTERING_HISTORY tells you what maintaining it costs.", "Cluster on low-to-moderate cardinality columns that match your WHERE clauses. Use expressions to coarsen high-cardinality keys.", "Point lookups on high-cardinality columns want Search Optimization, not a clustering key." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "RAG Over Warehouse Data: Text-to-SQL, Curated Marts, and the Guardrails Between Them", "summary": "Why raw text-to-SQL fails on real schemas, when retrieval over curated marts wins, and the semantic model and guardrails that make either safe.", "url": "https://datalane-data.blog/blog/rag-over-warehouse-data/", "published": "2026-04-12", "updated": "2026-04-12", "category": "AI & GenAI", "tags": [ "ai", "rag" ], "takeaways": [ "Text-to-SQL against a raw schema fails on join paths and metric definitions, not on SQL syntax.", "A semantic model is the contract: named metrics, allowed dimensions, and verified join paths the LLM cannot improvise around.", "Retrieval over curated marts beats SQL generation when questions are lookups, not aggregations.", "Every generated query runs under a read-only role with a query tag, a row limit, and a timeout. No exceptions.", "Evaluate with a golden set of question-to-answer pairs before anyone outside the team touches it." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "AWS Glue vs EMR for Spark: The Cost Model, the Cold Starts, and the Migration Point", "summary": "Glue vs EMR for production Spark: DPU pricing vs instance pricing, cold starts, library control, and the job size where teams switch.", "url": "https://datalane-data.blog/blog/aws-glue-vs-emr/", "published": "2026-04-11", "updated": "2026-04-11", "category": "AWS", "tags": [ "aws", "spark" ], "takeaways": [ "Glue bills per DPU-hour with a one-minute minimum. EMR bills for instances, so idle cluster time is your problem.", "Glue cold starts are real. Budget 30 to 90 seconds per job run before your first executor does anything.", "Library control is the migration trigger more often than cost. Native dependencies and custom Spark builds do not fit Glue.", "EMR on spot instances with managed scaling beats Glue on price once a job runs more than a few hours a day.", "Start on Glue, instrument DPU-hours from day one, and move the heavy jobs to EMR when the invoice or the dependencies force it." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "SQL Anti-Patterns That Quietly Multiply Your Warehouse Bill", "summary": "Five SQL habits that scan more than they should: SELECT *, functions on filter columns, DISTINCT as a bug fix, OR-joins, and per-row UDF calls.", "url": "https://datalane-data.blog/blog/sql-anti-patterns-performance/", "published": "2026-04-10", "updated": "2026-04-10", "category": "SQL", "tags": [ "sql", "performance" ], "takeaways": [ "SELECT * defeats column pruning on columnar warehouses. You pay for every column you did not read.", "A function on a filter column disables partition pruning. Rewrite the predicate so the raw column stands alone.", "DISTINCT that fixes a row count is hiding a fan-out join. Find the grain bug instead.", "An OR across two join conditions forces a near-cross-join. Split it into two joins or a UNION.", "A UDF called per row turns a set operation into a loop. Batch it or push the logic into SQL." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "A dbt Testing Strategy That Catches Regressions, Not 4,000 Warnings", "summary": "How I structure dbt tests in production: generic tests on keys, unit tests on logic, dbt-expectations on shape, severity that means something.", "url": "https://datalane-data.blog/blog/dbt-testing-strategy-production/", "published": "2026-04-09", "updated": "2026-04-09", "category": "dbt", "tags": [ "dbt", "quality" ], "takeaways": [ "A test that cannot fail a deploy is documentation. Decide which one you are writing before you write it.", "Generic tests belong on keys and grain: unique and not_null on the primary key of every mart, and almost nowhere else.", "Unit tests check SQL logic against fixed inputs. Data tests check production rows. Confusing the two is how suites rot.", "Severity, warn_if, and error_if turn a binary suite into a triage policy. Use them, or every failure becomes noise.", "store_failures is the difference between \"test failed\" and \"here are the 17 rows, go look.\"" ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Delta Lake Internals: The Transaction Log, Checkpoints, and Why Small Files Happen", "summary": "How the _delta_log actually works: JSON commits, optimistic concurrency, checkpoints, deletion vectors, and the small-file problem nobody escapes.", "url": "https://datalane-data.blog/blog/delta-lake-internals-transaction-log/", "published": "2026-04-08", "updated": "2026-04-08", "category": "Databricks", "tags": [ "databricks", "delta-lake" ], "takeaways": [ "A Delta table is Parquet files plus a _delta_log directory. The log is the table; the files are just data.", "Writers never lock. Optimistic concurrency means conflicts are detected at commit time, and blind appends almost never conflict.", "Checkpoints every 10 commits are why reads stay fast. A table with millions of commits and no checkpoint reads would be unusable.", "Deletion vectors turn DELETE and MERGE into bitmap writes instead of file rewrites, at the cost of read-time reconciliation.", "Small files come from streaming writers and over-parallel jobs. Auto compaction and optimized writes fix most of it before OPTIMIZE has to." ], "author": "Dinesh Chandra", "site": "DataLane" }, { "title": "Reading the Snowflake Query Profile: Where the Credits Actually Go", "summary": "How I read the operator tree: TableScan bytes, partition pruning, local and remote spilling, and the exchange steps that quietly dominate runtime.", "url": "https://datalane-data.blog/blog/snowflake-query-profile-guide/", "published": "2026-04-07", "updated": "2026-04-07", "category": "Snowflake", "tags": [ "snowflake", "performance" ], "takeaways": [ "The operator tree tells you where time went. The SQL text tells you almost nothing.", "Partitions scanned vs partitions total is the first number I read. Bad pruning is the most common root cause.", "Remote spilling means the warehouse is too small for the join or sort. Local spilling is a warning, remote is a bill.", "A fat exchange between operators usually means a join exploded or a bad distribution, not a slow scan.", "GET_QUERY_OPERATOR_STATS lets you read profiles in SQL, so you can hunt regressions across a whole workload." ], "author": "Dinesh Chandra", "site": "DataLane" } ] }