diff --git a/notebooks/pyiceberg_example.ipynb b/notebooks/basic_pyiceberg_example.ipynb similarity index 100% rename from notebooks/pyiceberg_example.ipynb rename to notebooks/basic_pyiceberg_example.ipynb diff --git a/notebooks/integration_duckdb_example.ipynb b/notebooks/integration_duckdb_example.ipynb new file mode 100644 index 0000000000..a0946208c8 --- /dev/null +++ b/notebooks/integration_duckdb_example.ipynb @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "# Libraries\n", + "import os\n", + "import tempfile\n", + "\n", + "import duckdb\n", + "import pyarrow as pa\n", + "import pyarrow.compute as pc\n", + "\n", + "from pyiceberg.catalog.sql import SqlCatalog\n", + "\n", + "# Create temporary folders for the warehouse and catalog\n", + "warehouse_path = tempfile.mkdtemp(prefix=\"iceberg_warehouse_\")\n", + "catalog_path = os.path.join(warehouse_path, \"catalog.db\")\n", + "print(\"Temporary warehouse:\", warehouse_path)\n", + "print(\"Temporary catalog:\", catalog_path)\n", + "\n", + "# Create a temporary SQL catalog using SQLite\n", + "catalog = SqlCatalog(name=\"tmp_sql_catalog\", uri=f\"sqlite:///{catalog_path}\", warehouse=f\"file://{warehouse_path}\", properties={})\n", + "# Create the default namespace\n", + "catalog.create_namespace(\"default\")" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "## First snapshot\n", + "We create the initial dataset and save it to an Iceberg table to create the first snapshot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Initial dataset\n", + "data1 = {\n", + " \"vendor_id\": [1, 2, 1, 2, 1],\n", + " \"trip_distance\": [1.5, 2.3, 0.8, 5.2, 3.1],\n", + " \"fare_amount\": [10.0, 15.5, 6.0, 22.0, 18.0],\n", + " \"tip_amount\": [2.0, 3.0, 1.0, 4.5, 3.5],\n", + " \"passenger_count\": [1, 2, 1, 3, 2],\n", + "}\n", + "df1 = pa.table(data1)\n", + "\n", + "# Create the Iceberg table and append initial data (first snapshot)\n", + "table = catalog.create_table(\"default.sample_trips\", schema=df1.schema)\n", + "table.append(df1)\n", + "print(\"First snapshot rows:\", len(table.scan().to_arrow()))" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Second snapshot\n", + "We add new data to the same table, creating a second snapshot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# New dataset for the second snapshot\n", + "data2 = {\n", + " \"vendor_id\": [3, 1],\n", + " \"trip_distance\": [2.0, 1.0],\n", + " \"fare_amount\": [12.0, 8.0],\n", + " \"tip_amount\": [1.5, 2.0],\n", + " \"passenger_count\": [1, 1],\n", + "}\n", + "df2 = pa.table(data2)\n", + "\n", + "# Append new data to the table (second snapshot)\n", + "table.append(df2)\n", + "print(\"Second snapshot total rows:\", len(table.scan().to_arrow()))" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## Compare snapshots using DuckDB\n", + "We load both snapshots into DuckDB as temporary tables to find added and removed rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Get snapshot IDs\n", + "snapshots = table.snapshots()\n", + "first_id = snapshots[0].snapshot_id\n", + "second_id = snapshots[-1].snapshot_id\n", + "print(\"Snapshot IDs:\", first_id, second_id)\n", + "\n", + "# Load snapshots into PyArrow tables\n", + "arrow_first = table.scan(snapshot_id=first_id).to_arrow()\n", + "arrow_second = table.scan(snapshot_id=second_id).to_arrow()\n", + "\n", + "# Connect to DuckDB and register tables\n", + "con = duckdb.connect()\n", + "con.register(\"first_snap\", arrow_first)\n", + "con.register(\"second_snap\", arrow_second)\n", + "\n", + "# Find added rows in the second snapshot\n", + "added_rows = con.execute(\"\"\"\n", + "SELECT * FROM second_snap\n", + "EXCEPT\n", + "SELECT * FROM first_snap\n", + "\"\"\").fetchall()\n", + "\n", + "# Find removed rows compared to the first snapshot\n", + "removed_rows = con.execute(\"\"\"\n", + "SELECT * FROM first_snap\n", + "EXCEPT\n", + "SELECT * FROM second_snap\n", + "\"\"\").fetchall()\n", + "\n", + "print(\"=== ADDED ROWS ===\")\n", + "for r in added_rows:\n", + " print(r)\n", + "\n", + "print(\"\\n=== REMOVED ROWS ===\")\n", + "for r in removed_rows:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## Filters and aggregations on the second snapshot\n", + "We add a computed column and perform filtering and aggregation using DuckDB." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Add computed column 'tip_per_mile'\n", + "arrow_second = arrow_second.append_column(\"tip_per_mile\", pc.divide(arrow_second[\"tip_amount\"], arrow_second[\"trip_distance\"]))\n", + "con.register(\"second_snap\", arrow_second)\n", + "\n", + "# Filter rows with tip_per_mile > 1.0\n", + "filtered_df = con.execute(\"SELECT * FROM second_snap WHERE tip_per_mile > 1.0\").fetchdf()\n", + "print(\"Filtered rows (tip_per_mile > 1.0):\")\n", + "print(filtered_df)\n", + "\n", + "# Aggregate total fare by vendor\n", + "agg_df = con.execute(\"SELECT vendor_id, SUM(fare_amount) AS total_fare FROM second_snap GROUP BY vendor_id\").fetchdf()\n", + "print(\"Total fare per vendor:\")\n", + "print(agg_df)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/spark_integration_example.ipynb b/notebooks/integration_spark_example.ipynb similarity index 100% rename from notebooks/spark_integration_example.ipynb rename to notebooks/integration_spark_example.ipynb