Feature | Add SqlBulkCopyOptions.CacheMetadata flag

doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml

@@ -235,6 +235,19 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only.
 ]]></format>
       </example>
     </Close>
+    <InvalidateMetadataCache>
+      <summary>
+        Clears the cached destination table metadata when using the <see cref="F:Microsoft.Data.SqlClient.SqlBulkCopyOptions.CacheMetadata" /> option.
+      </summary>
+      <remarks>
+        <para>
+          Call this method when you know the destination table schema has changed and you want to force the next <see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.WriteToServer(System.Data.Common.DbDataReader)" /> operation to refresh the metadata from the server.
+        </para>
+        <para>
+          The cache is automatically invalidated when the <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" /> property is changed to a different table name.
+        </para>
+      </remarks>
+    </InvalidateMetadataCache>
     <EnableStreaming>
       <summary>
         Enables or disables a <see cref="T:Microsoft.Data.SqlClient.SqlBulkCopy" /> object to stream data from an <see cref="T:System.Data.IDataReader" /> object

doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml

@@ -68,5 +68,19 @@ To see how the option changes the way the bulk load works, run the sample with t
         When specified, each batch of the bulk-copy operation will occur within a transaction. If you indicate this option and also provide a <see cref="T:Microsoft.Data.SqlClient.SqlTransaction" /> object to the constructor, an <see cref="T:System.ArgumentException" /> occurs.
       </summary>
     </UseInternalTransaction>
+    <CacheMetadata>
+      <summary>
+        <para>
+          When specified, <b>CacheMetadata</b> caches destination table metadata after the first bulk copy operation, allowing subsequent operations to the same table to skip the metadata discovery query. This can improve performance when performing multiple bulk copy operations to the same destination table.
+        </para>
+        <para>
+          <b>Warning:</b> Use this option only when you are certain the destination table schema will not change between bulk copy operations. If the table schema changes (columns added, removed, or modified), using cached metadata may result in data corruption, failed operations, or unexpected behavior. Call <see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.InvalidateMetadataCache" /> to clear the cache if the schema changes.
+        </para>
+        <para>
+          The cache is automatically invalidated when <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" /> is changed to a different table.
+          Changing <see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.ColumnMappings" /> between operations does not require cache invalidation because the cached metadata describes only the destination table schema, not the source-to-destination column mapping.
+        </para>
+      </summary>
+    </CacheMetadata>
   </members>
 </docs>

src/Microsoft.Data.SqlClient/netcore/ref/Microsoft.Data.SqlClient.cs

@@ -212,6 +212,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy
         public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
         public void Close() { }
+        /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/InvalidateMetadataCache/*'/>
+        public void InvalidateMetadataCache() { }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/System.IDisposable.Dispose/*'/>
         void System.IDisposable.Dispose() { }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/WriteToServer[@name="DbDataReaderParameter"]/*'/>
@@ -338,6 +340,8 @@ public enum SqlBulkCopyOptions
     {
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
         AllowEncryptedValueModifications = 64,
+        /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
+        CacheMetadata = 128,
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CheckConstraints/*'/>
         CheckConstraints = 2,
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/Default/*'/>

src/Microsoft.Data.SqlClient/netfx/ref/Microsoft.Data.SqlClient.cs

@@ -128,6 +128,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy
         public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
         public void Close() { }
+        /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/InvalidateMetadataCache/*'/>
+        public void InvalidateMetadataCache() { }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/System.IDisposable.Dispose/*'/>
         void System.IDisposable.Dispose() { }
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/WriteToServer[@name="DbDataReaderParameter"]/*'/>
@@ -254,6 +256,8 @@ public enum SqlBulkCopyOptions
     {
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
         AllowEncryptedValueModifications = 64,
+        /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
+        CacheMetadata = 128,
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CheckConstraints/*'/>
         CheckConstraints = 2,
         /// <include file='../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/Default/*'/>

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs

@@ -235,6 +235,14 @@ private int RowNumber
 
         private SourceColumnMetadata[] _currentRowMetadata;
 
+        // Metadata caching fields for CacheMetadata option
+        private BulkCopySimpleResultSet _cachedMetadata;
+        private string _cachedDestinationTableName;
+        // Per-operation clone of the destination table metadata, used when CacheMetadata is
+        // enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not
+        // mutate the cached BulkCopySimpleResultSet.
+        private _SqlMetaDataSet _operationMetaData;
+
 #if DEBUG
         internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task
         internal static bool SetAlwaysTaskOnWrite
@@ -353,6 +361,14 @@ public string DestinationTableName
                 {
                     throw ADP.ArgumentOutOfRange(nameof(DestinationTableName));
                 }
+
+                // Invalidate cached metadata if the destination table name changes
+                if (!string.Equals(_destinationTableName, value, StringComparison.Ordinal))
+                {
+                    _cachedMetadata = null;
+                    _cachedDestinationTableName = null;
+                }
+
                 _destinationTableName = value;
             }
         }
@@ -497,6 +513,16 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy
         // We need to have a _parser.RunAsync to make it real async.
         private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result)
         {
+            // Check if we have valid cached metadata for the current destination table
+            if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) &&
+                _cachedMetadata != null &&
+                string.Equals(_cachedDestinationTableName, _destinationTableName, StringComparison.Ordinal))
+            {
+                SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName);
+                result = _cachedMetadata;
+                return null;
+            }
+
             string TDSCommand = CreateInitialQuery();
             SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Initial Query: '{0}'", TDSCommand);
             SqlClientEventSource.Log.TryCorrelationTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Correlation | Object Id {0}, Activity Id {1}", ObjectID, ActivityCorrelator.Current);
@@ -506,6 +532,7 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
             {
                 result = new BulkCopySimpleResultSet();
                 RunParser(result);
+                CacheMetadataIfEnabled(result);
                 return null;
             }
             else
@@ -523,12 +550,23 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
                     {
                         var internalResult = new BulkCopySimpleResultSet();
                         RunParserReliably(internalResult);
+                        CacheMetadataIfEnabled(internalResult);
                         return internalResult;
                     }
                 }, TaskScheduler.Default);
             }
         }
 
+        private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result)
+        {
+            if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata))
+            {
+                _cachedMetadata = result;
+                _cachedDestinationTableName = _destinationTableName;
+                SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName);
+            }
+        }
+
         // Matches associated columns with metadata from initial query.
         // Builds and executes the update bulk command.
         private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults)
@@ -578,7 +616,17 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i
             bool appendComma = false;
 
             // Loop over the metadata for each result column.
+            // When using cached metadata, clone the metadata set so that null-pruning of
+            // unmatched/rejected columns does not mutate the shared cache. Without this,
+            // changing ColumnMappings between WriteToServer calls (e.g. mapping fewer columns
+            // on the first call, then more on the second) would permanently lose metadata
+            // entries from the cache.
             _SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData;
+            if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata) && _cachedMetadata != null)
+            {
+                metaDataSet = metaDataSet.Clone();
+            }
+            _operationMetaData = metaDataSet;
             _sortedColumnMappings = new List<_ColumnMapping>(metaDataSet.Length);
             for (int i = 0; i < metaDataSet.Length; i++)
             {
@@ -875,11 +923,19 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults)
         {
             _stateObj.SetTimeoutSeconds(BulkCopyTimeout);
 
-            _SqlMetaDataSet metadataCollection = internalResults[MetaDataResultId].MetaData;
+            _SqlMetaDataSet metadataCollection = _operationMetaData ?? internalResults[MetaDataResultId].MetaData;
             _stateObj._outputMessageType = TdsEnums.MT_BULK;
             _parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj);
         }
 
+        /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/InvalidateMetadataCache/*'/>
+        public void InvalidateMetadataCache()
+        {
+            _cachedMetadata = null;
+            _cachedDestinationTableName = null;
+            SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.InvalidateMetadataCache | Info | Metadata cache invalidated");
+        }
+
         // Terminates the bulk copy operation.
         // Must be called at the end of the bulk copy session.
         /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
@@ -900,6 +956,9 @@ private void Dispose(bool disposing)
                 // Dispose dependent objects
                 _columnMappings = null;
                 _parser = null;
+                _cachedMetadata = null;
+                _cachedDestinationTableName = null;
+                _operationMetaData = null;
                 try
                 {
                     // Just in case there is a lingering transaction (which there shouldn't be)
@@ -2667,7 +2726,7 @@ private Task CopyBatchesAsyncContinued(BulkCopySimpleResultSet internalResults,
 
                 // Load encryption keys now (if needed)
                 _parser.LoadColumnEncryptionKeys(
-                    internalResults[MetaDataResultId].MetaData,
+                    _operationMetaData ?? internalResults[MetaDataResultId].MetaData,
                     _connection);
 
                 Task task = CopyRowsAsync(0, _savedBatchSize, cts); // This is copying 1 batch of rows and setting _hasMoreRowToCopy = true/false.
@@ -3194,6 +3253,7 @@ private void ResetWriteToServerGlobalVariables()
             _dataTableSource = null;
             _dbDataReaderRowSource = null;
             _isAsyncBulkCopy = false;
+            _operationMetaData = null;
             _rowEnumerator = null;
             _rowSource = null;
             _rowSourceType = ValueSourceType.Unspecified;

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs

@@ -33,6 +33,9 @@ public enum SqlBulkCopyOptions
 
         /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
         AllowEncryptedValueModifications = 1 << 6,
+
+        /// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
+        CacheMetadata = 1 << 7,
     }
 }
 

src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlBulkCopyCacheMetadataTest.cs

@@ -0,0 +1,97 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.Tests
+{
+    public class SqlBulkCopyCacheMetadataTest
+    {
+        [Fact]
+        public void CacheMetadata_FlagValue_IsCorrect()
+        {
+            Assert.Equal(1 << 7, (int)SqlBulkCopyOptions.CacheMetadata);
+        }
+
+        [Fact]
+        public void CacheMetadata_CanBeCombinedWithOtherOptions()
+        {
+            SqlBulkCopyOptions combined =
+                SqlBulkCopyOptions.CacheMetadata |
+                SqlBulkCopyOptions.KeepIdentity |
+                SqlBulkCopyOptions.TableLock;
+
+            Assert.True((combined & SqlBulkCopyOptions.CacheMetadata) == SqlBulkCopyOptions.CacheMetadata);
+            Assert.True((combined & SqlBulkCopyOptions.KeepIdentity) == SqlBulkCopyOptions.KeepIdentity);
+            Assert.True((combined & SqlBulkCopyOptions.TableLock) == SqlBulkCopyOptions.TableLock);
+        }
+
+        [Fact]
+        public void CacheMetadata_DoesNotOverlapExistingFlags()
+        {
+            int cacheMetadataValue = (int)SqlBulkCopyOptions.CacheMetadata;
+            Assert.NotEqual((int)SqlBulkCopyOptions.Default, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.KeepIdentity, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.CheckConstraints, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.TableLock, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.KeepNulls, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.FireTriggers, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.UseInternalTransaction, cacheMetadataValue);
+            Assert.NotEqual((int)SqlBulkCopyOptions.AllowEncryptedValueModifications, cacheMetadataValue);
+        }
+
+        [Fact]
+        public void InvalidateMetadataCache_CanBeCalledWithoutError()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection());
+            bulkCopy.InvalidateMetadataCache();
+        }
+
+        [Fact]
+        public void InvalidateMetadataCache_CanBeCalledMultipleTimes()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection());
+            bulkCopy.InvalidateMetadataCache();
+            bulkCopy.InvalidateMetadataCache();
+            bulkCopy.InvalidateMetadataCache();
+        }
+
+        [Fact]
+        public void InvalidateMetadataCache_WithCacheMetadataOption()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null);
+            bulkCopy.InvalidateMetadataCache();
+        }
+
+        [Fact]
+        public void InvalidateMetadataCache_WithoutCacheMetadataOption()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.Default, null);
+            bulkCopy.InvalidateMetadataCache();
+        }
+
+        [Fact]
+        public void DestinationTableName_Change_DoesNotThrowWithCacheMetadata()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null);
+            bulkCopy.DestinationTableName = "Table1";
+            bulkCopy.DestinationTableName = "Table2";
+            bulkCopy.DestinationTableName = "Table1";
+        }
+
+        [Fact]
+        public void Constructor_WithCacheMetadataOption_Succeeds()
+        {
+            using SqlBulkCopy bulkCopy = new(new SqlConnection(), SqlBulkCopyOptions.CacheMetadata, null);
+            Assert.NotNull(bulkCopy);
+        }
+
+        [Fact]
+        public void Constructor_WithCacheMetadataAndConnectionString_Succeeds()
+        {
+            using SqlBulkCopy bulkCopy = new("Server=localhost", SqlBulkCopyOptions.CacheMetadata);
+            Assert.NotNull(bulkCopy);
+        }
+    }
+}

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj

@@ -111,6 +111,7 @@
     <Compile Include="SQL\SqlBulkCopyTest\Bug85007.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Bug903514.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\Bug98182.cs" />
+    <Compile Include="SQL\SqlBulkCopyTest\CacheMetadata.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\CheckConstraints.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\CopyWidenNullInexactNumerics.cs" />
     <Compile Include="SQL\SqlBulkCopyTest\DataConversionErrorMessageTest.cs" />