diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index f738939fe2..d5ca06455a 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -116,6 +116,10 @@ parameters: type: boolean default: false + # The SA password to set when configuring SQL Server. + - name: saPassword + type: string + jobs: - job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }} @@ -179,6 +183,7 @@ jobs: - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml@self # update config.json file parameters: debug: ${{ parameters.debug }} + saPassword: ${{ parameters.saPassword }} UseManagedSNIOnWindows: ${{ parameters.usemanagedSNI }} ${{ if parameters.configProperties.TCPConnectionString }}: TCPConnectionString: ${{ parameters.configProperties.TCPConnectionString }} @@ -255,6 +260,7 @@ jobs: parameters: operatingSystem: ${{ parameters.operatingSystem }} netcoreVersionTestUtils: ${{ parameters.netcoreVersionTestUtils }} + saPassword: ${{ parameters.saPassword }} ${{ if parameters.configProperties.instanceName }}: instanceName: ${{ parameters.configProperties.instanceName }} ${{ if parameters.configProperties.user }}: diff --git a/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml b/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml index a6e7755c6b..30d284f166 100644 --- a/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml +++ b/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml @@ -46,6 +46,8 @@ jobs: - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml parameters: + # We use the Library $(Password) variable as the SA password in this pipeline. + saPassword: $(Password) TCPConnectionString: $(SQL_TCP_CONN_STRING) NPConnectionString: $(SQL_NP_CONN_STRING) SupportsIntegratedSecurity: false diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index defba05d9f..0a0fd164e5 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -3,6 +3,9 @@ # The .NET Foundation licenses this file to you under the MIT license. # # See the LICENSE file in the project root for more information. # ################################################################################# + +# This stage depends on the secrets_stage. + parameters: - name: abstractionsArtifactsName type: string @@ -10,6 +13,10 @@ parameters: - name: abstractionsPackageVersion type: string + - name: additionalDependsOn + type: object + default: [] + - name: buildConfiguration type: string values: @@ -20,10 +27,6 @@ parameters: type: boolean default: false - - name: dependsOn - type: object - default: [] - - name: mdsArtifactsName type: string default: MDS.Artifacts @@ -56,7 +59,16 @@ stages: - ${{ each config in parameters.testConfigurations }}: - ${{ each image in config.value.images }}: - stage: ${{ image.key }} - dependsOn: ${{ parameters.dependsOn }} + dependsOn: + - secrets_stage + - ${{ each dep in parameters.additionalDependsOn }}: + - ${{ dep }} + + variables: + # Bring the SA password from the secrets_stage into scope here. + - name: saPassword + value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] + jobs: - ${{ each targetFramework in config.value.TargetFrameworks }}: - ${{ each platform in config.value.buildPlatforms }}: @@ -87,6 +99,7 @@ stages: configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} isArm64: ${{ eq(config.value.isArm64, 'true') }} + saPassword: $(saPassword) ${{if ne(config.value.configProperties, '{}') }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: ${{ if eq(x86TF, targetFramework) }}: @@ -123,6 +136,7 @@ stages: configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} isArm64: ${{ eq(config.value.isArm64, 'true') }} + saPassword: $(saPassword) ${{if and(eq(usemanagedSNI, false), ne(config.value.configProperties, '{}')) }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: ${{ if eq(x86TF, targetFramework) }}: diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index bcff695073..15de459d50 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -7,54 +7,51 @@ # This step configures an existing SQL Server running on the local Linux host. For example, our 1ES # Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and # running. -# -# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO -# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: - - name: condition + + # The SA password to set when configuring SQL Server. + - name: saPassword type: string - default: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) steps: -# Linux only steps -- bash: | - sudo systemctl stop mssql-server - - # Password for the SA user (required) - - MSSQL_SA_PW="$(Password)" - - # Product ID of the version of SQL server you're installing - # Must be evaluation, developer, express, web, standard, enterprise, or your 25 digit product key - MSSQL_PID="enterprise" - - echo Running mssql-conf setup... - sudo MSSQL_SA_PASSWORD="$MSSQL_SA_PW" \ - MSSQL_PID="$MSSQL_PID" \ - /opt/mssql/bin/mssql-conf -n setup accept-eula - - # Connect to server and get the version: - counter=1 - errstatus=1 - while [ $counter -le 5 ] && [ $errstatus = 1 ] - do - echo Waiting for SQL Server to start... - sleep 3s - /opt/mssql-tools/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P $MSSQL_SA_PW\ - -Q "SELECT @@VERSION" 2>/dev/null - errstatus=$? - ((counter++)) - done - - # Display error if connection failed: - if [ $errstatus = 1 ] - then - echo Cannot connect to SQL Server, installation aborted - exit $errstatus - fi - displayName: 'Configure SQL Server [Linux]' - condition: ${{parameters.condition }} + + # Configure SQL Server. + - bash: | + sudo systemctl stop mssql-server + + # Password for the SA user (required) + MSSQL_SA_PW="${{ parameters.saPassword }}" + + # Product ID of the version of SQL server you're installing + # Must be evaluation, developer, express, web, standard, enterprise, or your 25 digit product key + MSSQL_PID="enterprise" + + echo Running mssql-conf setup... + sudo MSSQL_SA_PASSWORD="$MSSQL_SA_PW" \ + MSSQL_PID="$MSSQL_PID" \ + /opt/mssql/bin/mssql-conf -n setup accept-eula + + # Connect to server and get the version: + counter=1 + errstatus=1 + while [ $counter -le 5 ] && [ $errstatus = 1 ] + do + echo Waiting for SQL Server to start... + sleep 3s + /opt/mssql-tools/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$MSSQL_SA_PW" \ + -Q "SELECT @@VERSION" 2>/dev/null + errstatus=$? + ((counter++)) + done + + # Display error if connection failed: + if [ $errstatus = 1 ] + then + echo Cannot connect to SQL Server, installation aborted + exit $errstatus + fi + displayName: 'Configure SQL Server [Linux]' diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index d41c9f25d7..0be4de6e2b 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -6,120 +6,119 @@ # This step installs the latest SQL Server 2022 onto the macOS host as a docker container and # configures it for use. -# -# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO -# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: - - name: condition + + # The SA password to set when configuring SQL Server. + - name: saPassword type: string - default: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) steps: -# macOS only steps -- bash: | - # The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it - # back to the pipeline user. - USER=`whoami` - SQLCMD_ERRORS=$(Agent.TempDirectory)/sqlcmd_err.log - echo "Errors will be written to: $SQLCMD_ERRORS" + # Configure SQL Server. This includes installing Docker and the SQLCMD tools, starting a SQL + # Server container, and verifying we can connect to it. + - bash: | + # The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it + # back to the pipeline user. + USER=`whoami` - # Configure the prompt to show the current timestamp so we can see how long each command takes. - export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' - set -x + SQLCMD_ERRORS=$(Agent.TempDirectory)/sqlcmd_err.log + echo "Errors will be written to: $SQLCMD_ERRORS" - # Install Docker and SQLCMD tools. - brew install colima - brew install --cask docker - brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release - brew update - HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18 - colima start --arch x86_64 - docker --version - docker pull mcr.microsoft.com/mssql/server:2022-latest + # Configure the prompt to show the current timestamp so we can see how long each command takes. + export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' + set -x - # Password for the SA user (required) - MSSQL_SA_PW="$(Password)" + # Install Docker and SQLCMD tools. + brew install colima + brew install --cask docker + brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release + brew update + HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18 + colima start --arch x86_64 + docker --version + docker pull mcr.microsoft.com/mssql/server:2022-latest - docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest + # Password for the SA user (required) + MSSQL_SA_PW="${{ parameters.saPassword }}" - sleep 5 + docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest - docker ps -a + sleep 5 - # Connect to the SQL Server container and get its version. - # - # It can take a while for the docker container to start listening and be - # ready for connections, so we will wait for up to 2 minutes, checking every - # 3 seconds. + docker ps -a - # Wait 3 seconds between attempts. - delay=3 + # Connect to the SQL Server container and get its version. + # + # It can take a while for the docker container to start listening and be + # ready for connections, so we will wait for up to 2 minutes, checking every + # 3 seconds. - # Try up to 40 times (2 minutes) to connect. - maxAttempts=40 + # Wait 3 seconds between attempts. + delay=3 - # Attempt counter. - attempt=1 + # Try up to 40 times (2 minutes) to connect. + maxAttempts=40 - # Flag to indicate when SQL Server is ready to accept connections. - ready=0 + # Attempt counter. + attempt=1 - while [ $attempt -le $maxAttempts ] - do + # Flag to indicate when SQL Server is ready to accept connections. + ready=0 - echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + while [ $attempt -le $maxAttempts ] + do - sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 + echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." - # If the command was successful, then the SQL Server is ready. - if [ $? -eq 0 ]; then - ready=1 - break - fi + sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 + + # If the command was successful, then the SQL Server is ready. + if [ $? -eq 0 ]; then + ready=1 + break + fi + + # Increment the attempt counter. + ((attempt++)) - # Increment the attempt counter. - ((attempt++)) + # Wait before trying again. + sleep $delay - # Wait before trying again. - sleep $delay + done - done + # Is the SQL Server ready? + if [ $ready -eq 0 ] + then + # No, so report the error(s) and exit. + echo Cannot connect to SQL Server; installation aborted; errors were: + cat $SQLCMD_ERRORS + rm -f $SQLCMD_ERRORS + exit 1 + fi - # Is the SQL Server ready? - if [ $ready -eq 0 ] - then - # No, so report the error(s) and exit. - echo Cannot connect to SQL Server; installation aborted; errors were: - cat $SQLCMD_ERRORS rm -f $SQLCMD_ERRORS - exit 1 - fi - - rm -f $SQLCMD_ERRORS - - echo "Use sqlcmd to show which IP addresses are being listened on..." - echo 0.0.0.0 - sqlcmd -S 0.0.0.0 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2 - echo 127.0.0.1 - sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2 - echo ::1 - sqlcmd -S ::1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2 - echo localhost - sqlcmd -S localhost -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2 - echo "(sqlcmd default / not specified)" - sqlcmd -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2 - - echo "Configuring Dedicated Administer Connections to allow remote connections..." - sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;" - if [ $? = 1 ] - then - echo "Error configuring DAC for remote access." - exit $errstatus - else - echo "Configuration complete." - fi - - displayName: 'Configure SQL Server [macOS]' - condition: ${{parameters.condition }} + + echo "Use sqlcmd to show which IP addresses are being listened on..." + echo 0.0.0.0 + sqlcmd -S 0.0.0.0 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + echo 127.0.0.1 + sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + echo ::1 + sqlcmd -S ::1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + echo localhost + sqlcmd -S localhost -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + echo "(sqlcmd default / not specified)" + sqlcmd -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + + echo "Configuring Dedicated Administer Connections to allow remote connections..." + sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;" + if [ $? = 1 ] + then + echo "Error configuring DAC for remote access." + exit $errstatus + else + echo "Configuration complete." + fi + + displayName: 'Configure SQL Server [macOS]' diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-step.yml index 822070e5ca..605b7311a6 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-step.yml @@ -17,6 +17,11 @@ parameters: type: string default: $(saUser) + # The SA password to set when configuring SQL Server. This will also be used for user accounts + # if necessary. + - name: saPassword + type: string + - name: SQLRootPath type: string default: '' @@ -76,6 +81,7 @@ steps: instanceName: ${{parameters.instanceName}} user: ${{parameters.user}} saUser: ${{parameters.saUser}} + saPassword: ${{parameters.saPassword}} SQLRootPath: ${{parameters.SQLRootPath}} fileStreamDirectory: ${{parameters.fileStreamDirectory}} x64AliasRegistryPath: ${{parameters.x64AliasRegistryPath}} @@ -89,10 +95,14 @@ steps: - ${{ elseif eq(parameters.operatingSystem, 'Linux') }}: # Linux only steps - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self + parameters: + saPassword: ${{ parameters.saPassword }} - ${{ elseif eq(parameters.operatingSystem, 'Mac') }}: # macOS only steps - template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self + parameters: + saPassword: ${{ parameters.saPassword }} # Common steps - task: DotNetCoreCLI@2 diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 50d8275969..749c30a79a 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -7,12 +7,9 @@ # This step configures an existing SQL Server running on the local Windows host. For example, our # 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and # running. -# -# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO -# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: -# Windows only parameters + - name: instanceName type: string default: MSSQLSERVER @@ -25,6 +22,10 @@ parameters: type: string default: $(saUser) + # The SA password to set when configuring SQL Server. This is also used for the user password. + - name: saPassword + type: string + - name: SQLRootPath type: string default: '' @@ -61,101 +62,54 @@ parameters: type: string default: $(LocalDbSharedInstanceName) -# Common parameters - - name: condition - type: string - default: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - steps: -# Windows only steps -- powershell: | - try - { - # Acquire a WMI handle. - Import-Module "sqlps" - $smo = 'Microsoft.SqlServer.Management.Smo.' - $wmi = new-object ($smo + 'Wmi.ManagedComputer') - - # List the WMI, including the instance names. - Write-Host "WMI Information:" - $wmi - - # Enable the TCP protocol on the default instance. - $Tcp = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Tcp']") - $Tcp.IsEnabled = $true - $Tcp.Alter() - - # Emit the TCP object to the pipeline log. - Write-Host "TCP Information:" - $Tcp - - # Enable the NP protocol on the default instance. - $Np = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Np']") - $Np.IsEnabled = $true - $Np.Alter() - - # Emit the NP object to the pipeline log. - Write-Host "NP Information:" - $Np - } - catch - { - $error[0] | format-list -force - throw - } - - New-NetFirewallRule -DisplayName "SQL TCP Ports" -Direction Inbound -Protocol TCP -LocalPort 1433 -Action allow - $sqlSrvPath = (Get-WmiObject win32_service | ?{$_.DisplayName -eq 'SQL Server (${{parameters.instanceName }})'} | select @{Name="Path"; Expression={$_.PathName.split('"')[1]}}).Path - New-NetFirewallRule -DisplayName "sqlservr.exe" -Program "$sqlSrvPath" - displayName: 'Enable TCP, NP & Firewall [Win]' - condition: ${{parameters.condition }} - retryCountOnTaskFailure: 2 - -- powershell: | - $password = "$(Password)" - - $machineName = $env:COMPUTERNAME - - if ("${{parameters.instanceName }}" -ne "MSSQLSERVER"){ - $machineName += "\${{parameters.instanceName }}" - } - - Write-Host $machineName - Import-Module "sqlps" - $tries = 0 - while ($true) { - $tries++ - try { - Invoke-Sqlcmd -ServerInstance "$machineName" @" - CREATE LOGIN [${{parameters.user }}] WITH PASSWORD=N'$password', - DEFAULT_DATABASE=[master], DEFAULT_LANGUAGE=[us_english], CHECK_EXPIRATION=OFF, CHECK_POLICY=OFF; - CREATE USER [${{parameters.user }}] FROM LOGIN [${{parameters.user }}]; - ALTER SERVER ROLE [sysadmin] ADD MEMBER [${{parameters.user }}]; - ALTER LOGIN [${{parameters.saUser }}] ENABLE; - ALTER LOGIN [${{parameters.saUser }}] WITH PASSWORD = '$password'; - "@ - break - } catch { - if ($tries -ge 5) { - Write-Host "##[error]Failed to create database user after $tries tries." - break - } - Write-Host "Failed to connect to server. Retrying in 5 seconds..." - Start-Sleep -Seconds 5 - } - } - displayName: 'Create SQL user [Win]' - condition: ${{parameters.condition }} - env: - SQL_USER: ${{parameters.user }} - SQL_PASSWD: $(Password) - -- ${{ if ne(parameters.SQLRootPath, '') }}: + + # GOTCHA: We must use the Windows-only powershell task here instead of the cross-platform pwsh + # task because we call some Windows-specific cmdlets. - powershell: | - #Enable FileStream - $instance = "${{parameters.instanceName }}" - $wmi = Get-WmiObject -Namespace "${{parameters.SQLRootPath }}" -Class FilestreamSettings | where {$_.InstanceName -eq $instance} - $wmi.EnableFilestream(3, $instance) + try + { + # Acquire a WMI handle. + Import-Module "sqlps" + $smo = 'Microsoft.SqlServer.Management.Smo.' + $wmi = new-object ($smo + 'Wmi.ManagedComputer') + + # List the WMI, including the instance names. + Write-Host "WMI Information:" + $wmi + + # Enable the TCP protocol on the default instance. + $Tcp = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Tcp']") + $Tcp.IsEnabled = $true + $Tcp.Alter() + + # Emit the TCP object to the pipeline log. + Write-Host "TCP Information:" + $Tcp + + # Enable the NP protocol on the default instance. + $Np = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Np']") + $Np.IsEnabled = $true + $Np.Alter() + + # Emit the NP object to the pipeline log. + Write-Host "NP Information:" + $Np + } + catch + { + $error[0] | format-list -force + throw + } + + New-NetFirewallRule -DisplayName "SQL TCP Ports" -Direction Inbound -Protocol TCP -LocalPort 1433 -Action allow + $sqlSrvPath = (Get-WmiObject win32_service | ?{$_.DisplayName -eq 'SQL Server (${{parameters.instanceName }})'} | select @{Name="Path"; Expression={$_.PathName.split('"')[1]}}).Path + New-NetFirewallRule -DisplayName "sqlservr.exe" -Program "$sqlSrvPath" + displayName: 'Enable TCP, NP & Firewall [Win]' + retryCountOnTaskFailure: 2 + + - powershell: | + $password = "${{ parameters.saPassword }}" $machineName = $env:COMPUTERNAME @@ -163,135 +117,170 @@ steps: $machineName += "\${{parameters.instanceName }}" } - #Change the access level for FileStream for SQLServer - Set-ExecutionPolicy Unrestricted + Write-Host $machineName Import-Module "sqlps" - Invoke-Sqlcmd -ServerInstance "$machineName" @" - EXEC sp_configure filestream_access_level, 2; - RECONFIGURE; + $tries = 0 + while ($true) { + $tries++ + try { + Invoke-Sqlcmd -ServerInstance "$machineName" @" + CREATE LOGIN [${{parameters.user }}] WITH PASSWORD=N'$password', + DEFAULT_DATABASE=[master], DEFAULT_LANGUAGE=[us_english], CHECK_EXPIRATION=OFF, CHECK_POLICY=OFF; + CREATE USER [${{parameters.user }}] FROM LOGIN [${{parameters.user }}]; + ALTER SERVER ROLE [sysadmin] ADD MEMBER [${{parameters.user }}]; + ALTER LOGIN [${{parameters.saUser }}] ENABLE; + ALTER LOGIN [${{parameters.saUser }}] WITH PASSWORD = '$password'; "@ - displayName: 'Enable FileStream [Win]' - condition: ${{parameters.condition }} + break + } catch { + if ($tries -ge 5) { + Write-Host "##[error]Failed to create database user after $tries tries." + break + } + Write-Host "Failed to connect to server. Retrying in 5 seconds..." + Start-Sleep -Seconds 5 + } + } + displayName: 'Create SQL user [Win]' env: SQL_USER: ${{parameters.user }} - SQL_PASSWD: $(Password) + SQL_PASSWD: ${{ parameters.saPassword }} -- ${{ if ne(parameters.FileStreamDirectory, '') }}: - - powershell: | - New-Item -Path ${{ parameters.fileStreamDirectory }} -ItemType Directory - displayName: 'Create FileStreamFolder' - retryCountOnTaskFailure: 1 - condition: ${{parameters.condition }} - continueOnError: true - -- powershell: | - $SQLServerName = ("{0}" -f [System.Net.Dns]::GetHostByName($env:computerName).HostName) - Write-Host FQDN is: $SQLServerName - - if ((Test-Path -Path ${{parameters.x64AliasRegistryPath }}) -ne $true) { - New-Item ${{parameters.x64AliasRegistryPath }} - } - - if ((Test-Path -Path ${{parameters.x86AliasRegistryPath }}) -ne $true) { - New-Item ${{parameters.x86AliasRegistryPath }} - } - - $TCPAliasName = "DBMSSOCN, $SQLServerName, ${{parameters.SQLAliasPort }}" - - New-ItemProperty -Path ${{parameters.x86AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName - New-ItemProperty -Path ${{parameters.x64AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName - displayName: 'Setup SQL Alias [Win]' - condition: ${{parameters.condition }} - -- powershell: | - # Create Certificate - $computerDnsName = [System.Net.Dns]::Resolve($null).HostName - $certificate = New-SelfSignedCertificate -DnsName $computerDnsName,localhost -CertStoreLocation cert:\LocalMachine\My -FriendlyName test99 -KeySpec KeyExchange - - # Get path to Private key (used later) - $keyPath = $certificate.PrivateKey.CspKeyContainerInfo.UniqueKeyContainerName - $machineKeyPath = "$env:ProgramData\Microsoft\Crypto\RSA\MachineKeys\$keyPath" - - # Add certificate to trusted roots - $store = new-object System.Security.Cryptography.X509Certificates.X509Store( - [System.Security.Cryptography.X509Certificates.StoreName]::Root, - "localmachine" - ) - - $store.open("MaxAllowed") - $store.add($certificate) - $store.close() - - # Get SQL Server instances and add the Certificate - $instances = Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\Instance Names\SQL' - foreach ($instance in $instances){ - $instance | ForEach-Object { - $_.PSObject.Properties | Where-Object { $_.Name -notmatch '^PS.*' } | ForEach-Object { - Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\$($_.Value)\MSSQLServer\SuperSocketNetLib" -Name Certificate -Value $certificate.Thumbprint.ToLower() - - # Grant read access to Private Key for SQL Service Account - if ($($_.Name) -eq "MSSQLSERVER") { - icacls $machineKeyPath /grant "NT Service\MSSQLSERVER:R" - } else { - icacls $machineKeyPath /grant "NT Service\MSSQL`$$($_.Name):R" - } - } - } - } - displayName: 'Add SQL Certificate [Win]' - condition: ${{parameters.condition }} - -- powershell: | - # You need to restart SQL Server for the change to persist - # -Force takes care of any dependent services, like SQL Agent. - # Note: if the instance is named, replace MSSQLSERVER with MSSQL$ followed by - # the name of the instance (e.g. MSSQL$MYINSTANCE) - - $serviceName = "${{parameters.instanceName }}" - $InstancePrefix = 'MSSQL$' - - if ( "${{parameters.instanceName }}" -ne "MSSQLSERVER" ) - { - $serviceName = $InstancePrefix+"${{parameters.instanceName }}" - } - - Restart-Service -Name "$serviceName" -Force - Restart-Service -Name MSSQLSERVER* -Force - - displayName: 'Restart SQL Server [Win]' - condition: ${{parameters.condition }} - -- powershell: | - $arrService = Get-Service -Name "SQLBrowser" - $arrService - - if ($arrService.Status -eq 'Stopped') { - Write-Host 'Attempt to run the service ...' - # updating the startup type to make sure it's not disabled - Set-Service -StartupType Automatic $arrService.Name - $arrService.Start() - - $arrService.WaitForStatus('Running', '00:00:30') - if ($arrService.Status -eq 'Running') { - $arrService - } else { - Write-Error 'Timed out waiting for service to start.' + - ${{ if ne(parameters.SQLRootPath, '') }}: + - powershell: | + #Enable FileStream + $instance = "${{parameters.instanceName }}" + $wmi = Get-WmiObject -Namespace "${{parameters.SQLRootPath }}" -Class FilestreamSettings | where {$_.InstanceName -eq $instance} + $wmi.EnableFilestream(3, $instance) + + $machineName = $env:COMPUTERNAME + + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER"){ + $machineName += "\${{parameters.instanceName }}" } - } - displayName: 'Start Sql Server Browser [Win]' - condition: ${{parameters.condition }} -- ${{ if parameters.enableLocalDB }}: + #Change the access level for FileStream for SQLServer + Set-ExecutionPolicy Unrestricted + Import-Module "sqlps" + Invoke-Sqlcmd -ServerInstance "$machineName" @" + EXEC sp_configure filestream_access_level, 2; + RECONFIGURE; + "@ + displayName: 'Enable FileStream [Win]' + env: + SQL_USER: ${{parameters.user }} + SQL_PASSWD: ${{ parameters.saPassword }} + + - ${{ if ne(parameters.FileStreamDirectory, '') }}: + - powershell: | + New-Item -Path ${{ parameters.fileStreamDirectory }} -ItemType Directory + displayName: 'Create FileStreamFolder' + retryCountOnTaskFailure: 1 + continueOnError: true + - powershell: | - #script to enable local db - - SqlLocalDB info - #SqlLocalDB create ${{parameters.localDbAppName }} - SqlLocalDB info ${{parameters.localDbAppName }} - SqlLocalDB share ${{parameters.localDbAppName }} ${{parameters.LocalDbSharedInstanceName }} - SqlLocalDB start ${{parameters.localDbAppName }} - SqlLocalDB info ${{parameters.localDbAppName }} - - sqlcmd -S "(localdb)\.\${{parameters.LocalDbSharedInstanceName }}" -q "SELECT @@VERSION" - displayName: 'Enable LocalDB [Win]' - condition: ${{parameters.condition }} + $SQLServerName = ("{0}" -f [System.Net.Dns]::GetHostByName($env:computerName).HostName) + Write-Host FQDN is: $SQLServerName + + if ((Test-Path -Path ${{parameters.x64AliasRegistryPath }}) -ne $true) { + New-Item ${{parameters.x64AliasRegistryPath }} + } + + if ((Test-Path -Path ${{parameters.x86AliasRegistryPath }}) -ne $true) { + New-Item ${{parameters.x86AliasRegistryPath }} + } + + $TCPAliasName = "DBMSSOCN, $SQLServerName, ${{parameters.SQLAliasPort }}" + + New-ItemProperty -Path ${{parameters.x86AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName + New-ItemProperty -Path ${{parameters.x64AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName + displayName: 'Setup SQL Alias [Win]' + + - powershell: | + # Create Certificate + $computerDnsName = [System.Net.Dns]::Resolve($null).HostName + $certificate = New-SelfSignedCertificate -DnsName $computerDnsName,localhost -CertStoreLocation cert:\LocalMachine\My -FriendlyName test99 -KeySpec KeyExchange + + # Get path to Private key (used later) + $keyPath = $certificate.PrivateKey.CspKeyContainerInfo.UniqueKeyContainerName + $machineKeyPath = "$env:ProgramData\Microsoft\Crypto\RSA\MachineKeys\$keyPath" + + # Add certificate to trusted roots + $store = new-object System.Security.Cryptography.X509Certificates.X509Store( + [System.Security.Cryptography.X509Certificates.StoreName]::Root, + "localmachine" + ) + + $store.open("MaxAllowed") + $store.add($certificate) + $store.close() + + # Get SQL Server instances and add the Certificate + $instances = Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\Instance Names\SQL' + foreach ($instance in $instances){ + $instance | ForEach-Object { + $_.PSObject.Properties | Where-Object { $_.Name -notmatch '^PS.*' } | ForEach-Object { + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\$($_.Value)\MSSQLServer\SuperSocketNetLib" -Name Certificate -Value $certificate.Thumbprint.ToLower() + + # Grant read access to Private Key for SQL Service Account + if ($($_.Name) -eq "MSSQLSERVER") { + icacls $machineKeyPath /grant "NT Service\MSSQLSERVER:R" + } else { + icacls $machineKeyPath /grant "NT Service\MSSQL`$$($_.Name):R" + } + } + } + } + displayName: 'Add SQL Certificate [Win]' + + - powershell: | + # You need to restart SQL Server for the change to persist + # -Force takes care of any dependent services, like SQL Agent. + # Note: if the instance is named, replace MSSQLSERVER with MSSQL$ followed by + # the name of the instance (e.g. MSSQL$MYINSTANCE) + + $serviceName = "${{parameters.instanceName }}" + $InstancePrefix = 'MSSQL$' + + if ( "${{parameters.instanceName }}" -ne "MSSQLSERVER" ) + { + $serviceName = $InstancePrefix+"${{parameters.instanceName }}" + } + + Restart-Service -Name "$serviceName" -Force + Restart-Service -Name MSSQLSERVER* -Force + + displayName: 'Restart SQL Server [Win]' + + - powershell: | + $arrService = Get-Service -Name "SQLBrowser" + $arrService + + if ($arrService.Status -eq 'Stopped') { + Write-Host 'Attempt to run the service ...' + # updating the startup type to make sure it's not disabled + Set-Service -StartupType Automatic $arrService.Name + $arrService.Start() + + $arrService.WaitForStatus('Running', '00:00:30') + if ($arrService.Status -eq 'Running') { + $arrService + } else { + Write-Error 'Timed out waiting for service to start.' + } + } + displayName: 'Start Sql Server Browser [Win]' + + - ${{ if parameters.enableLocalDB }}: + - powershell: | + #script to enable local db + + SqlLocalDB info + #SqlLocalDB create ${{parameters.localDbAppName }} + SqlLocalDB info ${{parameters.localDbAppName }} + SqlLocalDB share ${{parameters.localDbAppName }} ${{parameters.LocalDbSharedInstanceName }} + SqlLocalDB start ${{parameters.localDbAppName }} + SqlLocalDB info ${{parameters.localDbAppName }} + + sqlcmd -S "(localdb)\.\${{parameters.LocalDbSharedInstanceName }}" -q "SELECT @@VERSION" + displayName: 'Enable LocalDB [Win]' diff --git a/eng/pipelines/common/templates/steps/update-config-file-step.yml b/eng/pipelines/common/templates/steps/update-config-file-step.yml index 65ac9ecf08..fad1bc1636 100644 --- a/eng/pipelines/common/templates/steps/update-config-file-step.yml +++ b/eng/pipelines/common/templates/steps/update-config-file-step.yml @@ -8,6 +8,10 @@ parameters: type: boolean default: false + # The SA password to set when configuring SQL Server. + - name: saPassword + type: string + - name: TCPConnectionString type: string default: '' @@ -35,11 +39,11 @@ parameters: - name: TracingEnabled type: boolean default: false - + - name: AADAuthorityURL type: string default: '' - + - name: AADPasswordConnectionString type: string default: '' @@ -72,15 +76,15 @@ parameters: type: string default: '' - - name: LocalDbAppName + - name: LocalDbAppName type: string default: '' - - name: LocalDbSharedInstanceName + - name: LocalDbSharedInstanceName type: string default: '' - - name: AliasName + - name: AliasName type: string default: '' @@ -92,19 +96,19 @@ parameters: type: boolean default: false - - name: DNSCachingConnString + - name: DNSCachingConnString type: string default: '' - - name: DNSCachingServerCR + - name: DNSCachingServerCR type: string default: '' - - name: DNSCachingServerTR + - name: DNSCachingServerTR type: string default: '' - - name: EnclaveAzureDatabaseConnString + - name: EnclaveAzureDatabaseConnString type: string default: '' @@ -129,76 +133,93 @@ parameters: default: '' steps: -# All properties should be added here, and this template should be used for any manipulation of the config.json file. -- pwsh: | - $jdata = Get-Content -Raw "config.default.json" | ConvertFrom-Json - foreach ($p in $jdata) - { - $p.TCPConnectionString="${{parameters.TCPConnectionString }}" - $p.NPConnectionString="${{parameters.NPConnectionString }}" + # Some of the connection strings brought in from Azure DevOps Library groups expect a runtime + # $(Password) variable to exist. This is used for username/password logins to SQL Servers running + # locally on the agent. We must set $(Password) here so it is available when those connection + # strings are expanded. We cannot use a Library variable because it will be deemed a secret, and + # will not be available to forked repos. + # + # We use the saPassword parameter as the value for this variable. + # + # All subsequent steps in the current job will expand $(Password) with the saPassword parameter's + # value. + # + - pwsh: | + $password = "${{ parameters.saPassword }}" + Write-Host "##vso[task.setvariable variable=Password;isSecret=true]$password" + displayName: Set Connection String Password + + # All properties should be added here, and this template should be used for any manipulation of the config.json file. + - pwsh: | + $jdata = Get-Content -Raw "config.default.json" | ConvertFrom-Json + foreach ($p in $jdata) + { + $p.TCPConnectionString="${{parameters.TCPConnectionString }}" - $p.AADAuthorityURL="${{parameters.AADAuthorityURL }}" + $p.NPConnectionString="${{parameters.NPConnectionString }}" - $p.AADPasswordConnectionString="${{parameters.AADPasswordConnectionString }}" + $p.AADAuthorityURL="${{parameters.AADAuthorityURL }}" - $p.AADServicePrincipalId="${{parameters.AADServicePrincipalId }}" + $p.AADPasswordConnectionString="${{parameters.AADPasswordConnectionString }}" - $p.AADServicePrincipalSecret="${{parameters.AADServicePrincipalSecret }}" + $p.AADServicePrincipalId="${{parameters.AADServicePrincipalId }}" - $p.AzureKeyVaultUrl="${{parameters.AzureKeyVaultUrl }}" + $p.AADServicePrincipalSecret="${{parameters.AADServicePrincipalSecret }}" - $p.AzureKeyVaultTenantId="${{parameters.AzureKeyVaultTenantId }}" + $p.AzureKeyVaultUrl="${{parameters.AzureKeyVaultUrl }}" - $p.UserManagedIdentityClientId="${{parameters.UserManagedIdentityClientId }}" + $p.AzureKeyVaultTenantId="${{parameters.AzureKeyVaultTenantId }}" - $p.FileStreamDirectory="${{parameters.FileStreamDirectory }}" + $p.UserManagedIdentityClientId="${{parameters.UserManagedIdentityClientId }}" - $p.LocalDbSharedInstanceName="${{parameters.LocalDbSharedInstanceName }}" + $p.FileStreamDirectory="${{parameters.FileStreamDirectory }}" - $p.AliasName="${{parameters.AliasName }}" + $p.LocalDbSharedInstanceName="${{parameters.LocalDbSharedInstanceName }}" - $p.EnclaveAzureDatabaseConnString="${{parameters.EnclaveAzureDatabaseConnString }}" + $p.AliasName="${{parameters.AliasName }}" - $p.DNSCachingServerTR="${{parameters.DNSCachingServerTR }}" + $p.EnclaveAzureDatabaseConnString="${{parameters.EnclaveAzureDatabaseConnString }}" - $p.DNSCachingServerCR="${{parameters.DNSCachingServerCR }}" + $p.DNSCachingServerTR="${{parameters.DNSCachingServerTR }}" - $p.DNSCachingConnString="${{parameters.DNSCachingConnString }}" + $p.DNSCachingServerCR="${{parameters.DNSCachingServerCR }}" - $p.SupportsFileStream="${{parameters.SupportsFileStream }}" + $p.DNSCachingConnString="${{parameters.DNSCachingConnString }}" - $p.LocalDbAppName="${{parameters.LocalDbAppName }}" + $p.SupportsFileStream="${{parameters.SupportsFileStream }}" - $p.TCPConnectionStringAASSGX="${{parameters.TCPConnectionStringAASSGX }}" + $p.LocalDbAppName="${{parameters.LocalDbAppName }}" - $p.TCPConnectionStringNoneVBS="${{parameters.TCPConnectionStringNoneVBS }}" + $p.TCPConnectionStringAASSGX="${{parameters.TCPConnectionStringAASSGX }}" - $p.TCPConnectionStringHGSVBS="${{parameters.TCPConnectionStringHGSVBS }}" + $p.TCPConnectionStringNoneVBS="${{parameters.TCPConnectionStringNoneVBS }}" - $p.UseManagedSNIOnWindows=[System.Convert]::ToBoolean("${{parameters.UseManagedSNIOnWindows }}") - $p.SupportsIntegratedSecurity=[System.Convert]::ToBoolean("${{parameters.SupportsIntegratedSecurity }}") - $p.ManagedIdentitySupported=[System.Convert]::ToBoolean("${{parameters.ManagedIdentitySupported }}") - $p.IsAzureSynapse=[System.Convert]::ToBoolean("${{parameters.IsAzureSynapse }}") - $p.IsDNSCachingSupportedTR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedTR }}") - $p.IsDNSCachingSupportedCR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedCR }}") - $p.TracingEnabled=[System.Convert]::ToBoolean("${{parameters.TracingEnabled }}") - $p.EnclaveEnabled=[System.Convert]::ToBoolean("${{parameters.EnclaveEnabled }}") - $p.WorkloadIdentityFederationServiceConnectionId="${{parameters.WorkloadIdentityFederationServiceConnectionId }}" - } - $jdata | ConvertTo-Json | Set-Content "config.json" - workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: 'Update config.json' + $p.TCPConnectionStringHGSVBS="${{parameters.TCPConnectionStringHGSVBS }}" -- ${{ if eq(parameters.debug, true) }}: - - pwsh: | - $jdata = Get-Content -Raw "config.json" | ConvertFrom-Json - foreach ($p in $jdata) - { - foreach ($prop in $p.PSObject.Properties) - { - Write-Host "Property: $($prop.Name) Value: $($prop.Value)" - } + $p.UseManagedSNIOnWindows=[System.Convert]::ToBoolean("${{parameters.UseManagedSNIOnWindows }}") + $p.SupportsIntegratedSecurity=[System.Convert]::ToBoolean("${{parameters.SupportsIntegratedSecurity }}") + $p.ManagedIdentitySupported=[System.Convert]::ToBoolean("${{parameters.ManagedIdentitySupported }}") + $p.IsAzureSynapse=[System.Convert]::ToBoolean("${{parameters.IsAzureSynapse }}") + $p.IsDNSCachingSupportedTR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedTR }}") + $p.IsDNSCachingSupportedCR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedCR }}") + $p.TracingEnabled=[System.Convert]::ToBoolean("${{parameters.TracingEnabled }}") + $p.EnclaveEnabled=[System.Convert]::ToBoolean("${{parameters.EnclaveEnabled }}") + $p.WorkloadIdentityFederationServiceConnectionId="${{parameters.WorkloadIdentityFederationServiceConnectionId }}" } + $jdata | ConvertTo-Json | Set-Content "config.json" workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: '[Debug] Emit config.json' + displayName: 'Update config.json' + + - ${{ if eq(parameters.debug, true) }}: + - pwsh: | + $jdata = Get-Content -Raw "config.json" | ConvertFrom-Json + foreach ($p in $jdata) + { + foreach ($prop in $p.PSObject.Properties) + { + Write-Host "Property: $($prop.Name) Value: $($prop.Value)" + } + } + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: '[Debug] Emit config.json' diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 801afa0fdd..72f803ca4b 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -122,6 +122,11 @@ variables: stages: + # Generate secrets used throughout the pipeline. + - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self + parameters: + debug: ${{ parameters.debug }} + # Build the Abstractions package, and publish it to the pipeline artifacts # under the given artifact name. - template: /eng/pipelines/stages/build-abstractions-package-ci-stage.yml@self @@ -173,7 +178,7 @@ stages: # When building via packages, we must depend on the Abstractions and MDS # packages. ${{ if eq(parameters.referenceType, 'Package') }}: - dependsOn: + additionalDependsOn: - build_abstractions_package_stage - build_mds_akv_packages_stage dotnetVerbosity: ${{ parameters.dotnetVerbosity }} @@ -186,7 +191,7 @@ stages: - template: /eng/pipelines/stages/stress-tests-ci-stage.yml@self parameters: buildConfiguration: ${{ parameters.buildConfiguration }} - dependsOn: + additionalDependsOn: - build_mds_akv_packages_stage - build_azure_package_stage mdsArtifactsName: $(mdsArtifactsName) @@ -209,7 +214,7 @@ stages: # When testing MDS via packages, we must depend on the Abstractions, # MDS, and Azure packages. ${{ if eq(parameters.referenceType, 'Package') }}: - dependsOn: + additionalDependsOn: - build_abstractions_package_stage - build_mds_akv_packages_stage - build_azure_package_stage @@ -442,10 +447,10 @@ stages: TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) NPConnectionString: $(AZURE_DB_NP_CONN_STRING) AADAuthorityURL: $(AADAuthorityURL) - # Note: Using the isFork variable to determine if secrets are available is not ideal since - # it's an indirect association. But everything else (referencing secret variables various - # ways to detect if they were present) won't run consistently across forks and non-forks. - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: + # Pipeline runs against forks of the repo don't have access to Library secrets, so we + # omit them entirely from the configProperties, which causes the dependent tests to be + # skipped. + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) AADServicePrincipalSecret: $(AADServicePrincipalSecret) AADServicePrincipalId: $(AADServicePrincipalId) @@ -474,7 +479,7 @@ stages: TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING_eastus) NPConnectionString: $(AZURE_DB_NP_CONN_STRING_eastus) AADAuthorityURL: $(AADAuthorityURL) - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR_eastus) AADServicePrincipalSecret: $(AADServicePrincipalSecret) AADServicePrincipalId: $(AADServicePrincipalId) @@ -529,7 +534,7 @@ stages: TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) NPConnectionString: $(AZURE_DB_NP_CONN_STRING) AADAuthorityURL: $(AADAuthorityURL) - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) AADServicePrincipalSecret: $(AADServicePrincipalSecret) AADServicePrincipalId: $(AADServicePrincipalId) @@ -565,9 +570,9 @@ stages: # Enclave tests # - # Only run the AE tests if enable and if we have access to the necessary - # secrets. - ${{ if and(eq(parameters.runAlwaysEncryptedTests, true), eq(variables['system.pullRequest.isFork'], 'False')) }}: + # Only run these tests if explicitly enabled, and if we're not a forked repo (which won't + # have access to the necessary Library secrets). + ${{ if and(eq(parameters.runAlwaysEncryptedTests, true), eq(variables['System.PullRequest.IsFork'], 'False')) }}: windows_enclave_sql: pool: ADO-CI-AE-1ES-Pool images: @@ -588,8 +593,7 @@ stages: EnclaveEnabled: true AADAuthorityURL: $(AADAuthorityURL) AADServicePrincipalId: $(AADServicePrincipalId) - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: - AADServicePrincipalSecret: $(AADServicePrincipalSecret) + AADServicePrincipalSecret: $(AADServicePrincipalSecret) AzureKeyVaultUrl: $(AzureKeyVaultUrl) AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) SupportsIntegratedSecurity: true @@ -617,8 +621,7 @@ stages: TCPConnectionStringAASSGX: $(SQL_TCP_CONN_STRING_AASSGX) EnclaveEnabled: true AADServicePrincipalId: $(AADServicePrincipalId) - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: - AADServicePrincipalSecret: $(AADServicePrincipalSecret) + AADServicePrincipalSecret: $(AADServicePrincipalSecret) AzureKeyVaultUrl: $(AzureKeyVaultUrl) AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) SupportsIntegratedSecurity: false diff --git a/eng/pipelines/jobs/stress-tests-ci-job.yml b/eng/pipelines/jobs/stress-tests-ci-job.yml index 53de7e407f..7477295f54 100644 --- a/eng/pipelines/jobs/stress-tests-ci-job.yml +++ b/eng/pipelines/jobs/stress-tests-ci-job.yml @@ -39,13 +39,8 @@ parameters: default: '' # The pipeline step to run to configure SQL Server. - # - # This step is expected to require no parameters. It must configure a SQL - # Server instance listening on localhost for SQL auth via the 'sa' user with - # the pipeline variable $(Password) as the password. - name: sqlSetupStep - type: string - default: '' + type: step # The name of the MDS pipeline artifacts to download. - name: mdsArtifactsName @@ -83,9 +78,6 @@ parameters: default: [] # The stress test config file contents to write to the config file. - # - # This should point to the SQL Server configured via the sqlSetupStep - # parameter, with user 'sa' and password of $(Password). - name: configContent type: string default: '' @@ -143,7 +135,7 @@ jobs: targetPath: $(Build.SourcesDirectory)/packages # Setup the local SQL Server. - - template: ${{ parameters.sqlSetupStep }}@self + - ${{ parameters.sqlSetupStep }} # We use the 'custom' command because the DotNetCoreCLI@2 task doesn't support # all of our argument combinations for the different build steps. diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 2d2da14435..9b414e6eeb 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -97,6 +97,11 @@ parameters: - Package - Project + # The SA password set by the sqlServerSetupSteps, if relevant. + - name: saPassword + type: string + default: '' + # Steps to run, if any, to configure a local SQL Server instance on the agent # VM. - name: sqlServerSetupSteps @@ -222,6 +227,7 @@ jobs: - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml@self parameters: debug: ${{ parameters.debug }} + saPassword: ${{ parameters.saPassword }} # The config.json file has many options, but only some of them are # used by the Azure package tests. We only specify the ones that are @@ -238,12 +244,7 @@ jobs: # Avoid exposing secrets to pipeline jobs triggered via forks. This # prevents external contributors from creating PRs and running # pipelines that could expose these secrets. - # - # Note that this isn't a perfect restriction since internal - # contributors may want to use forks, but this would prevent them from - # running the full test suite. We don't have a better way to detect - # external parties though. - ${{ if eq(variables['system.pullRequest.isFork'], 'False') }}: + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) AADServicePrincipalSecret: $(AADServicePrincipalSecret) @@ -273,10 +274,10 @@ jobs: # List the DLLs in the output directory for debugging purposes. - ${{ if eq(parameters.debug, true) }}: - - pwsh: > + - pwsh: >- Get-ChildItem - -Path "src/Microsoft.Data.SqlClient.Extensions/Azure/test/bin/${{ parameters.buildConfiguration }}" - -Recurse + -Path "src/Microsoft.Data.SqlClient.Extensions/Azure/test/bin/${{ parameters.buildConfiguration }}" + -Recurse displayName: '[Debug] List Output DLLs' # Run the tests for each .NET runtime. diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index 658c5fa30b..c9f135f731 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -20,6 +20,8 @@ # The packages are published to pipeline artifacts with the name specified by # the azureArtifactsName parameter. # +# This stage depends on the secrets_stage. +# # This template defines a stage named 'build_azure_package_stage' that # can be depended on by downstream stages. @@ -38,6 +40,11 @@ parameters: - name: abstractionsPackageVersion type: string + # Additional stages we depend on, if any. + - name: additionalDependsOn + type: object + default: [] + # The name of the pool to use for jobs that require customized VM images. - name: adoPoolName type: string @@ -79,11 +86,6 @@ parameters: type: boolean default: false - # The stages we depend on, if any. - - name: dependsOn - type: object - default: [] - # The dotnet CLI verbosity to use. - name: dotnetVerbosity type: string @@ -124,7 +126,15 @@ stages: - stage: build_azure_package_stage displayName: Build Azure Package - dependsOn: ${{ parameters.dependsOn }} + dependsOn: + - secrets_stage + - ${{ each dep in parameters.additionalDependsOn }}: + - ${{ dep }} + + variables: + # Bring the SA password from the secrets_stage into scope here. + - name: saPassword + value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] jobs: @@ -164,13 +174,12 @@ stages: netRuntimes: [net8.0, net9.0, net10.0] poolName: ${{ parameters.adoPoolName }} referenceType: ${{ parameters.referenceType }} + saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. sqlServerSetupSteps: - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: - # Override the template's default step condition. We always - # want this step to run. - condition: true + saPassword: $(saPassword) vmImage: ADO-UB22-SQL22 # ------------------------------------------------------------------------ @@ -210,14 +219,13 @@ stages: netRuntimes: [net8.0, net9.0, net10.0] poolName: ${{ parameters.adoPoolName }} referenceType: ${{ parameters.referenceType }} + saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. sqlServerSetupSteps: - template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self # Use defaults for most parameters. parameters: - # Override the template's default step condition. We always - # want this step to run. - condition: true + saPassword: $(saPassword) enableLocalDB: true # These variables are from an Azure DevOps Library variable # group. diff --git a/eng/pipelines/stages/generate-secrets-ci-stage.yml b/eng/pipelines/stages/generate-secrets-ci-stage.yml new file mode 100644 index 0000000000..507e18d279 --- /dev/null +++ b/eng/pipelines/stages/generate-secrets-ci-stage.yml @@ -0,0 +1,70 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This stage generates the following random secrets for use throughout the PR and CI pipelines: +# +# SaPassword - A random GUID suitable for use as the SA password of local SQL Server instances. +# +# Subsequent stages may reference these variables as: +# +# $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] +# +# For further details on the stage-dependency syntax, see: +# +# https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#set-an-output-variable-for-use-in-future-stages +# +# Any stages that use these secrets must depend on this stage: +# +# secrets_stage +# +# None of the values produced here are actual secrets - they are just random values that are +# suitable for testing purposes, and do not need to be protected. For simplicity, they are emitted +# as regular output variables of script steps, are not formally marked as secrets, and not stored in +# the Azure DevOps Library or Key Vault. +# +parameters: + + # True to emit debugging steps and messages. + - name: debug + type: boolean + default: false + +stages: + + # The stage downstream stages must depend on to ensure the secrets are generated before they are + # used. + - stage: secrets_stage + displayName: Generate Secrets + jobs: + + # The job that generates the secrets. Each secret is emitted as an output variable of a + # script step, and can be referenced by downstream stages via the stage-dependencies syntax. + - job: secrets_job + displayName: Generate Secrets + pool: + # We don't need anything special, so use the standard Microsoft-hosted Ubuntu image, which + # is typically very fast to spin up. + vmImage: ubuntu-latest + + steps: + + # We don't need the repo checked out. + - checkout: none + + # Generate a password suitable for the SA user of local SQL Server instances. + # + # This creates the SaPassword.Value variable. + # + - bash: | + guid=$(cat /proc/sys/kernel/random/uuid) + echo "##vso[task.setvariable variable=Value;isOutput=true]$guid" + name: SaPassword + displayName: Generate SA password + + # Emit the SA password, if desired. + - ${{ if eq(parameters.debug, true) }}: + - bash: | + echo "SA password: $(SaPassword.Value)" + displayName: '[Debug] Emit SA password' diff --git a/eng/pipelines/stages/stress-tests-ci-stage.yml b/eng/pipelines/stages/stress-tests-ci-stage.yml index d1eae30ff4..87185509bf 100644 --- a/eng/pipelines/stages/stress-tests-ci-stage.yml +++ b/eng/pipelines/stages/stress-tests-ci-stage.yml @@ -12,15 +12,21 @@ # src/Microsoft.Data.SqlClient/tests/StressTests # # All tests use a localhost SQL Server configured for SQL auth via the 'sa' user -# and password of '$(Password)'. The $(Password) variable is defined in the ADO -# Library "ADO Test Configuration properties", brought in by -# common/templates/libraries/ci-build-variables.yml. +# and the provided password. +# +# This stage depends on the secrets_stage. # # This template defines a stage named 'run_stress_tests_stage' that can be # depended on by downstream stages. parameters: + # The names of any additional stages this stage depends on, for example the stages that publish + # the MDS package artifacts we will test. + - name: additionalDependsOn + type: object + default: [] + # The Azure package version to stress test. This version must be available in # one of the configured NuGet sources. - name: azurePackageVersion @@ -35,12 +41,6 @@ parameters: - Debug - Release - # The names of any stages this stage depends on, for example the stages - # that publish the MDS package artifacts we will test. - - name: dependsOn - type: object - default: [] - # The verbosity level for the dotnet CLI commands. - name: dotnetVerbosity type: string @@ -74,7 +74,10 @@ parameters: stages: - stage: run_stress_tests_stage displayName: Run Stress Tests - dependsOn: ${{ parameters.dependsOn }} + dependsOn: + - secrets_stage + - ${{ each dep in parameters.additionalDependsOn }}: + - ${{ dep }} variables: # The directory where dotnet artifacts will be staged. Not to be @@ -104,6 +107,10 @@ stages: $(commonArguments) --configuration ${{parameters.buildConfiguration}} + # Bring the SA password from the secrets_stage into scope here. + - name: saPassword + value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] + # The contents of the config file to use for all tests. We will write # this to a JSON file for each test job, and then point to it via the # STRESS_CONFIG_FILE environment variable. @@ -116,7 +123,7 @@ stages: "isDefault": true, "dataSource": "localhost", "user": "sa", - "password": "$(Password)", + "password": "$(saPassword)", "supportsWindowsAuthentication": false, "isLocal": false, "disableMultiSubnetFailover": true, @@ -136,7 +143,10 @@ stages: displayNamePrefix: Linux poolName: $(ci_var_defaultPoolName) vmImage: ADO-UB22-SQL22 - sqlSetupStep: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self + parameters: + saPassword: $(saPassword) mdsArtifactsName: ${{ parameters.mdsArtifactsName }} solution: $(solution) testProject: $(testProject) @@ -156,7 +166,10 @@ stages: # The Windows images include a suitable .NET Framework runtime, so we # don't have to install one explicitly. vmImage: ADO-MMS22-SQL22 - sqlSetupStep: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self + parameters: + saPassword: $(saPassword) mdsArtifactsName: ${{ parameters.mdsArtifactsName }} solution: $(solution) testProject: $(testProject) @@ -179,7 +192,10 @@ stages: # generic one from Azure Pipelines. poolName: Azure Pipelines vmImage: macos-latest - sqlSetupStep: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self + parameters: + saPassword: $(saPassword) mdsArtifactsName: ${{ parameters.mdsArtifactsName }} solution: $(solution) testProject: $(testProject) diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs index 86c876c068..f153096d9f 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs @@ -143,6 +143,13 @@ static Config() DebugEmit = GetEnvFlag("TEST_DEBUG_EMIT"); SystemAccessToken = GetEnvVar("SYSTEM_ACCESSTOKEN"); + // Circumvent pipeline masking of things it considers secrets by base64 encoding them. We + // will emit them on their own line without labels. + string Base64Encode(string s) + { + return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(s)); + } + // Emit debug information if requested. if (DebugEmit) { @@ -157,22 +164,30 @@ static Config() $" ManagedIdentitySupported: {ManagedIdentitySupported}"); Console.WriteLine( $" PasswordConnectionString: {PasswordConnectionString}"); + Console.WriteLine( + $" {Base64Encode(PasswordConnectionString)}"); Console.WriteLine( $" ServicePrincipalId: {ServicePrincipalId}"); Console.WriteLine( - $" ServicePrincipalSecret: {ServicePrincipalSecret.Length}"); + $" ServicePrincipalSecret: {ServicePrincipalSecret}"); + Console.WriteLine( + $" {Base64Encode(ServicePrincipalSecret)}"); Console.WriteLine( $" SystemAccessToken: {SystemAccessToken}"); Console.WriteLine( $" SystemAssignedManagedIdentitySupported: {SystemAssignedManagedIdentitySupported}"); Console.WriteLine( $" TcpConnectionString: {TcpConnectionString}"); + Console.WriteLine( + $" {Base64Encode(TcpConnectionString)}"); Console.WriteLine( $" TenantId: {TenantId}"); Console.WriteLine( $" UseManagedSniOnWindows: {UseManagedSniOnWindows}"); Console.WriteLine( $" UserManagedIdentityClientId: {UserManagedIdentityClientId}"); + Console.WriteLine( + $" {Base64Encode(UserManagedIdentityClientId)}"); Console.WriteLine( " WorkloadIdentityFederationServiceConnectionId: " + WorkloadIdentityFederationServiceConnectionId);