From 1fbd97a461c1a950df76caebd6457ea25dc47b89 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Jan 2026 13:19:56 +0100 Subject: [PATCH 1/2] acc: invariant: support init/cleanup scripts; fix synced_database_table --- .../configs/synced_database_table.yml.tmpl | 3 ++- .../synced_database_table.yml.tmpl-cleanup.sh | 6 ++++++ .../synced_database_table.yml.tmpl-init.sh | 15 +++++++++++++++ acceptance/bundle/invariant/no_drift/script | 12 ++++++++++++ acceptance/bundle/invariant/test.toml | 9 +++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100755 acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-cleanup.sh create mode 100755 acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh diff --git a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl index ebb8161dc2..babb4d20b1 100644 --- a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl +++ b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl @@ -20,7 +20,8 @@ resources: database_instance_name: ${resources.database_instances.instance1.name} logical_database_name: ${resources.database_catalogs.catalog1.database_name} spec: - source_table_full_name: samples.nyctaxi.trips + # Use a unique source table per test run to avoid hitting the 20-table-per-source limit + source_table_full_name: main.test_synced_$UNIQUE_NAME.trips_source scheduling_policy: SNAPSHOT primary_key_columns: - tpep_pickup_datetime diff --git a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-cleanup.sh new file mode 100755 index 0000000000..4890ef1481 --- /dev/null +++ b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Clean up the temporary source table +echo "Cleaning up temporary source table" +$CLI tables delete main.test_synced_$UNIQUE_NAME.trips_source || true +$CLI schemas delete main.test_synced_$UNIQUE_NAME || true diff --git a/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh new file mode 100755 index 0000000000..4a47f8aa2b --- /dev/null +++ b/acceptance/bundle/invariant/configs/synced_database_table.yml.tmpl-init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Create a unique source table for this test run to avoid hitting the 20-table-per-source limit +echo "Creating temporary source table: main.test_synced_$UNIQUE_NAME.trips_source" + +# Create schema using CLI +$CLI schemas create test_synced_$UNIQUE_NAME main -o json | jq '{full_name}' + +# Create source table from samples.nyctaxi.trips using SQL API +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to C:/Program Files/Git/api/... +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.test_synced_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 16c51ec62e..a5d6967b19 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -4,6 +4,12 @@ # Copy data files to test directory cp -r "$TESTDIR/../data/." . &> LOG.cp +# Run init script if present +INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" +if [ -f "$INIT_SCRIPT" ]; then + source "$INIT_SCRIPT" &> LOG.init +fi + envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml cp databricks.yml LOG.config @@ -16,6 +22,12 @@ cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null cleanup() { trace $CLI bundle destroy --auto-approve &> LOG.destroy cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null + + # Run cleanup script if present + CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + if [ -f "$CLEANUP_SCRIPT" ]; then + source "$CLEANUP_SCRIPT" &> LOG.cleanup + fi } trap cleanup EXIT diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index d807c49f2a..01c885b02e 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -35,3 +35,12 @@ EnvMatrix.INPUT_CONFIG = [ "synced_database_table.yml.tmpl", "volume.yml.tmpl", ] + +# Fake SQL endpoint for local tests +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{name}" +Response.Body = '{"status": "OK"}' From 0a00fe91b44230ca14166d9698886a5d1b0a1d7b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Jan 2026 14:42:03 +0100 Subject: [PATCH 2/2] update resources.yml --- bundle/direct/dresources/resources.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index fa67266054..9edf8de46e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -109,6 +109,13 @@ resources: # database_instances: no special config - # database_catalogs: no special config + database_catalogs: + ignore_remote_changes: + # Backend does not set this: + - create_database_if_not_exists - # synced_database_tables: no special config + synced_database_tables: + ignore_remote_changes: + # Backend does not set these fields in response (it sets effective_ counterparts instead) + - database_instance_name + - logical_database_name