From badcd036ba6c853820471e92f6b5b507c7808bbb Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:49:07 +0100 Subject: [PATCH 01/14] feat(identities): create dedicated identity management module - Add resource group creation for bootstrap identities - Add user-assigned managed identity resources (map-based) - Add federated identity credential support for workload identity federation - Support for plan, apply, and future agent identities - Add comprehensive module documentation BREAKING CHANGE: Identity resources moved to separate module for better separation of concerns --- modules/identities/README.md | 48 +++++++++++++++++++++++++++++++++ modules/identities/main.tf | 26 ++++++++++++++++++ modules/identities/outputs.tf | 29 ++++++++++++++++++++ modules/identities/terraform.tf | 9 +++++++ modules/identities/variables.tf | 32 ++++++++++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 modules/identities/README.md create mode 100644 modules/identities/main.tf create mode 100644 modules/identities/outputs.tf create mode 100644 modules/identities/terraform.tf create mode 100644 modules/identities/variables.tf diff --git a/modules/identities/README.md b/modules/identities/README.md new file mode 100644 index 0000000..8ee7986 --- /dev/null +++ b/modules/identities/README.md @@ -0,0 +1,48 @@ +# Bootstrap Identities Module + +This module creates User Assigned Managed Identities and their federated credentials for use in the Azure Landing Zones (ALZ) bootstrap process. + +## Purpose + +This module manages all identity resources needed for the ALZ accelerator bootstrap: +- Resource group for identities +- Plan and Apply user-assigned managed identities (for Terraform state management) +- Federated identity credentials (for workload identity federation with Azure DevOps/GitHub) +- Optional agent identity (for Container App Jobs with UAMI authentication) + +## Usage + +```hcl +module "identities" { + source = "../../modules/identities" + + resource_group_name = "rg-identity-prod" + location = "eastus" + + managed_identities = { + plan = "uami-plan-prod" + apply = "uami-apply-prod" + } + + federated_credentials = { + plan = { + user_assigned_managed_identity_key = "plan" + federated_credential_subject = "sc://org/project/environment/plan" + federated_credential_issuer = "https://vstoken.dev.azure.com/..." + federated_credential_name = "fc-plan" + audience = ["api://AzureADTokenExchange"] + } + } + + tags = { + environment = "production" + } +} +``` + +## Outputs + +The module provides maps keyed by logical names for easy reference: +- `managed_identity_ids` - For passing to other modules +- `managed_identity_client_ids` - For authentication +- `managed_identity_principal_ids` - For role assignments diff --git a/modules/identities/main.tf b/modules/identities/main.tf new file mode 100644 index 0000000..06cbe8b --- /dev/null +++ b/modules/identities/main.tf @@ -0,0 +1,26 @@ +# Identity Resource Group +resource "azurerm_resource_group" "identity" { + name = var.resource_group_name + location = var.location + tags = var.tags +} + +# User Assigned Managed Identities +resource "azurerm_user_assigned_identity" "identities" { + for_each = var.managed_identities + name = each.value + location = var.location + resource_group_name = azurerm_resource_group.identity.name + tags = var.tags +} + +# Federated Identity Credentials (for workload identity federation) +resource "azurerm_federated_identity_credential" "credentials" { + for_each = var.federated_credentials + name = each.value.federated_credential_name + resource_group_name = azurerm_resource_group.identity.name + audience = each.value.audience + issuer = each.value.federated_credential_issuer + parent_id = azurerm_user_assigned_identity.identities[each.value.user_assigned_managed_identity_key].id + subject = each.value.federated_credential_subject +} diff --git a/modules/identities/outputs.tf b/modules/identities/outputs.tf new file mode 100644 index 0000000..397b089 --- /dev/null +++ b/modules/identities/outputs.tf @@ -0,0 +1,29 @@ +output "resource_group_name" { + value = azurerm_resource_group.identity.name + description = "Name of the identity resource group" +} + +output "resource_group_id" { + value = azurerm_resource_group.identity.id + description = "Resource ID of the identity resource group" +} + +output "managed_identity_ids" { + value = { for k, v in azurerm_user_assigned_identity.identities : k => v.id } + description = "Map of managed identity resource IDs (key = logical name)" +} + +output "managed_identity_client_ids" { + value = { for k, v in azurerm_user_assigned_identity.identities : k => v.client_id } + description = "Map of managed identity client IDs for authentication (key = logical name)" +} + +output "managed_identity_principal_ids" { + value = { for k, v in azurerm_user_assigned_identity.identities : k => v.principal_id } + description = "Map of managed identity principal IDs for role assignments (key = logical name)" +} + +output "federated_credential_ids" { + value = { for k, v in azurerm_federated_identity_credential.credentials : k => v.id } + description = "Map of federated credential IDs" +} diff --git a/modules/identities/terraform.tf b/modules/identities/terraform.tf new file mode 100644 index 0000000..5026d7c --- /dev/null +++ b/modules/identities/terraform.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.9" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + } +} diff --git a/modules/identities/variables.tf b/modules/identities/variables.tf new file mode 100644 index 0000000..c8cf043 --- /dev/null +++ b/modules/identities/variables.tf @@ -0,0 +1,32 @@ +variable "resource_group_name" { + type = string + description = "Name of the resource group for bootstrap identities" +} + +variable "location" { + type = string + description = "Azure location for resources" +} + +variable "managed_identities" { + type = map(string) + description = "Map of managed identities to create. Key is the logical name (e.g., 'plan', 'apply'), value is the resource name." +} + +variable "federated_credentials" { + type = map(object({ + user_assigned_managed_identity_key = string + federated_credential_subject = string + federated_credential_issuer = string + federated_credential_name = string + audience = list(string) + })) + default = {} + description = "Federated identity credentials for workload identity federation. Typically used with Azure DevOps or GitHub Actions." +} + +variable "tags" { + type = map(string) + default = {} + description = "Tags to apply to all resources" +} From 69eefb0138f7f82829909a4ef5228a0415beabf3 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:49:56 +0100 Subject: [PATCH 02/14] refactor(azure)!: consume identities from dedicated module - Change from creating identities to accepting them as inputs - Replace user_assigned_managed_identities with managed_identity_ids map - Replace federated_credentials with managed_identity_client_ids map - Add managed_identity_principal_ids for role assignments - Remove identity resource group creation - Delete managed_identity.tf (moved to identities module) - Update role assignments to use passed-in principal IDs - Update outputs to pass through identity client IDs BREAKING CHANGE: Module no longer creates identities internally --- modules/azure/managed_identity.tf | 16 ---------------- modules/azure/outputs.tf | 3 ++- modules/azure/resource_groups.tf | 6 ++---- modules/azure/role_assignments.tf | 2 +- modules/azure/variables.tf | 22 ++++++++++++---------- 5 files changed, 17 insertions(+), 32 deletions(-) delete mode 100644 modules/azure/managed_identity.tf diff --git a/modules/azure/managed_identity.tf b/modules/azure/managed_identity.tf deleted file mode 100644 index a0b1195..0000000 --- a/modules/azure/managed_identity.tf +++ /dev/null @@ -1,16 +0,0 @@ -resource "azurerm_user_assigned_identity" "alz" { - for_each = var.user_assigned_managed_identities - location = var.azure_location - name = each.value - resource_group_name = azurerm_resource_group.identity.name -} - -resource "azurerm_federated_identity_credential" "alz" { - for_each = var.federated_credentials - name = each.value.federated_credential_name - resource_group_name = azurerm_resource_group.identity.name - audience = [local.audience] - issuer = each.value.federated_credential_issuer - parent_id = azurerm_user_assigned_identity.alz[each.value.user_assigned_managed_identity_key].id - subject = each.value.federated_credential_subject -} diff --git a/modules/azure/outputs.tf b/modules/azure/outputs.tf index 33d7297..c4807f0 100644 --- a/modules/azure/outputs.tf +++ b/modules/azure/outputs.tf @@ -1,5 +1,6 @@ output "user_assigned_managed_identity_client_ids" { - value = { for key, value in var.user_assigned_managed_identities : key => azurerm_user_assigned_identity.alz[key].client_id } + value = var.managed_identity_client_ids + description = "Map of managed identity client IDs (passed through from identities module)" } output "role_assignments" { diff --git a/modules/azure/resource_groups.tf b/modules/azure/resource_groups.tf index 6b2877c..af98de5 100644 --- a/modules/azure/resource_groups.tf +++ b/modules/azure/resource_groups.tf @@ -4,10 +4,8 @@ resource "azurerm_resource_group" "state" { location = var.azure_location } -resource "azurerm_resource_group" "identity" { - name = var.resource_group_identity_name - location = var.azure_location -} +# Identity resource group is now created by the identities module +# Variable kept for backwards compatibility but resource removed resource "azurerm_resource_group" "agents" { count = var.use_self_hosted_agents ? 1 : 0 diff --git a/modules/azure/role_assignments.tf b/modules/azure/role_assignments.tf index 76a4e28..11ee545 100644 --- a/modules/azure/role_assignments.tf +++ b/modules/azure/role_assignments.tf @@ -3,7 +3,7 @@ locals { user_assigned_managed_identity_key = value.user_assigned_managed_identity_key custom_role_definition_key = value.custom_role_definition_key scope = value.scope - principal_id = azurerm_user_assigned_identity.alz[value.user_assigned_managed_identity_key].principal_id + principal_id = var.managed_identity_principal_ids[value.user_assigned_managed_identity_key] } } additional_role_assignments = { for assignment in flatten([ diff --git a/modules/azure/variables.tf b/modules/azure/variables.tf index 6c36bb9..9b45526 100644 --- a/modules/azure/variables.tf +++ b/modules/azure/variables.tf @@ -2,18 +2,20 @@ variable "azure_location" { type = string } -variable "user_assigned_managed_identities" { - type = map(string) +# Managed identities are now created externally and passed in +variable "managed_identity_ids" { + type = map(string) + description = "Map of managed identity resource IDs (key = logical name like 'plan', 'apply')" } -variable "federated_credentials" { - type = map(object({ - user_assigned_managed_identity_key = string - federated_credential_subject = string - federated_credential_issuer = string - federated_credential_name = string - })) - default = {} +variable "managed_identity_client_ids" { + type = map(string) + description = "Map of managed identity client IDs for outputs" +} + +variable "managed_identity_principal_ids" { + type = map(string) + description = "Map of managed identity principal IDs for role assignments" } variable "resource_group_identity_name" { From 2e082aedca1f8b191ef7b9a861fd0a797f76a0d2 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:50:16 +0100 Subject: [PATCH 03/14] feat(azuredevops): integrate identities module with orchestration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add module.identities instantiation with Azure DevOps federated credentials - Configure federated credentials with api://AzureADTokenExchange audience - Update module.azure to consume identity outputs - Update module.azure_devops to use identity client IDs - Add moved blocks for backwards-compatible state migration - Add tags variable support for identity resources Moved blocks ensure existing deployments upgrade without resource recreation: - module.azure.azurerm_resource_group.identity → module.identities.azurerm_resource_group.identity - module.azure.azurerm_user_assigned_identity.alz → module.identities.azurerm_user_assigned_identity.identities - module.azure.azurerm_federated_identity_credential.alz → module.identities.azurerm_federated_identity_credential.credentials --- alz/azuredevops/locals.tf | 5 +++- alz/azuredevops/main.tf | 55 +++++++++++++++++++++++++++++++++--- alz/azuredevops/variables.tf | 6 ++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/alz/azuredevops/locals.tf b/alz/azuredevops/locals.tf index 9ec0cf2..2448619 100644 --- a/alz/azuredevops/locals.tf +++ b/alz/azuredevops/locals.tf @@ -39,18 +39,21 @@ locals { (local.apply_key) = local.resource_names.user_assigned_managed_identity_apply } - federated_credentials = { + # Federated credentials for the identities module (includes audience) + federated_credentials_for_identities = { (local.plan_key) = { user_assigned_managed_identity_key = local.plan_key federated_credential_subject = module.azure_devops.subjects[local.plan_key] federated_credential_issuer = module.azure_devops.issuers[local.plan_key] federated_credential_name = local.resource_names.user_assigned_managed_identity_federated_credentials_plan + audience = ["api://AzureADTokenExchange"] } (local.apply_key) = { user_assigned_managed_identity_key = local.apply_key federated_credential_subject = module.azure_devops.subjects[local.apply_key] federated_credential_issuer = module.azure_devops.issuers[local.apply_key] federated_credential_name = local.resource_names.user_assigned_managed_identity_federated_credentials_apply + audience = ["api://AzureADTokenExchange"] } } diff --git a/alz/azuredevops/main.tf b/alz/azuredevops/main.tf index ced90ca..284e973 100644 --- a/alz/azuredevops/main.tf +++ b/alz/azuredevops/main.tf @@ -26,9 +26,25 @@ module "files" { additional_folders_path = var.additional_folders_path } +# ======================================== +# IDENTITY MANAGEMENT +# ======================================== +# Create all bootstrap identities (plan, apply) in a dedicated module +# This must happen before azure_devops module to get federated credentials + +module "identities" { + source = "../../modules/identities" + resource_group_name = local.resource_names.resource_group_identity + location = var.bootstrap_location + managed_identities = local.managed_identities + # Federated credentials created after azure_devops module provides subjects/issuers + federated_credentials = local.federated_credentials_for_identities + tags = var.tags +} + module "azure" { source = "../../modules/azure" - resource_group_identity_name = local.resource_names.resource_group_identity + resource_group_identity_name = module.identities.resource_group_name resource_group_agents_name = local.resource_names.resource_group_agents resource_group_network_name = local.resource_names.resource_group_network resource_group_state_name = local.resource_names.resource_group_state @@ -36,8 +52,9 @@ module "azure" { storage_account_name = local.resource_names.storage_account storage_container_name = local.resource_names.storage_container azure_location = var.bootstrap_location - user_assigned_managed_identities = local.managed_identities - federated_credentials = local.federated_credentials + managed_identity_ids = module.identities.managed_identity_ids + managed_identity_client_ids = module.identities.managed_identity_client_ids + managed_identity_principal_ids = module.identities.managed_identity_principal_ids agent_container_instances = local.agent_container_instances agent_container_instance_managed_identity_name = local.resource_names.container_instance_managed_identity agent_organization_url = module.azure_devops.organization_url @@ -84,7 +101,7 @@ module "azure_devops" { create_project = var.azure_devops_create_project project_name = var.azure_devops_project_name environments = local.environments - managed_identity_client_ids = module.azure.user_assigned_managed_identity_client_ids + managed_identity_client_ids = module.identities.managed_identity_client_ids repository_name = local.resource_names.version_control_system_repository repository_files = local.repository_files template_repository_files = local.template_repository_files @@ -104,3 +121,33 @@ module "azure_devops" { use_self_hosted_agents = var.use_self_hosted_agents create_branch_policies = var.create_branch_policies } + +# ======================================== +# STATE MIGRATION (Backwards Compatibility) +# ======================================== +# These moved blocks ensure existing deployments migrate smoothly to the new identity module structure + +moved { + from = module.azure.azurerm_resource_group.identity + to = module.identities.azurerm_resource_group.identity +} + +moved { + from = module.azure.azurerm_user_assigned_identity.alz["plan"] + to = module.identities.azurerm_user_assigned_identity.identities["plan"] +} + +moved { + from = module.azure.azurerm_user_assigned_identity.alz["apply"] + to = module.identities.azurerm_user_assigned_identity.identities["apply"] +} + +moved { + from = module.azure.azurerm_federated_identity_credential.alz["plan"] + to = module.identities.azurerm_federated_identity_credential.credentials["plan"] +} + +moved { + from = module.azure.azurerm_federated_identity_credential.alz["apply"] + to = module.identities.azurerm_federated_identity_credential.credentials["apply"] +} diff --git a/alz/azuredevops/variables.tf b/alz/azuredevops/variables.tf index 5f0accd..d0329f8 100644 --- a/alz/azuredevops/variables.tf +++ b/alz/azuredevops/variables.tf @@ -670,3 +670,9 @@ variable "storage_account_blob_versioning_enabled" { type = bool default = true } + +variable "tags" { + type = map(string) + default = {} + description = "Tags to apply to Azure resources" +} From a3030466b45958d971717638c69a51ff347b8ee5 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:50:35 +0100 Subject: [PATCH 04/14] feat(github): integrate identities module with orchestration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add module.identities instantiation with GitHub Actions federated credentials - Configure federated credentials for GitHub workload identity federation - Update module.azure to consume identity outputs - Update module.github to use identity client IDs - Add moved blocks for backwards-compatible state migration - Add tags variable support for identity resources Moved blocks ensure existing deployments upgrade without resource recreation: - module.azure.azurerm_resource_group.identity → module.identities.azurerm_resource_group.identity - module.azure.azurerm_user_assigned_identity.alz → module.identities.azurerm_user_assigned_identity.identities - module.azure.azurerm_federated_identity_credential.alz → module.identities.azurerm_federated_identity_credential.credentials --- alz/github/locals.tf | 4 +++- alz/github/main.tf | 49 +++++++++++++++++++++++++++++++++++++---- alz/github/variables.tf | 7 ++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/alz/github/locals.tf b/alz/github/locals.tf index 15f71dc..c612ab9 100644 --- a/alz/github/locals.tf +++ b/alz/github/locals.tf @@ -53,12 +53,14 @@ locals { (local.apply_key) = local.resource_names.user_assigned_managed_identity_apply } - federated_credentials = { for key, value in module.github.subjects : + # Federated credentials for the identities module (includes audience) + federated_credentials_for_identities = { for key, value in module.github.subjects : key => { user_assigned_managed_identity_key = value.user_assigned_managed_identity_key federated_credential_subject = value.subject federated_credential_issuer = module.github.issuer federated_credential_name = "${local.resource_names.user_assigned_managed_identity_federated_credentials_prefix}-${key}" + audience = ["api://AzureADTokenExchange"] } } diff --git a/alz/github/main.tf b/alz/github/main.tf index 7dfed0c..e7dde9e 100644 --- a/alz/github/main.tf +++ b/alz/github/main.tf @@ -26,11 +26,25 @@ module "files" { additional_folders_path = var.additional_folders_path } +# ======================================== +# IDENTITY MANAGEMENT +# ======================================== +# Create all bootstrap identities (plan, apply) in a dedicated module +# This must happen before github module to get federated credentials + +module "identities" { + source = "../../modules/identities" + resource_group_name = local.resource_names.resource_group_identity + location = var.bootstrap_location + managed_identities = local.managed_identities + # Federated credentials created after github module provides subjects/issuers + federated_credentials = local.federated_credentials_for_identities + tags = var.tags +} + module "azure" { source = "../../modules/azure" - user_assigned_managed_identities = local.managed_identities - federated_credentials = local.federated_credentials - resource_group_identity_name = local.resource_names.resource_group_identity + resource_group_identity_name = module.identities.resource_group_name resource_group_state_name = local.resource_names.resource_group_state resource_group_agents_name = local.resource_names.resource_group_agents resource_group_network_name = local.resource_names.resource_group_network @@ -38,6 +52,9 @@ module "azure" { storage_account_name = local.resource_names.storage_account storage_container_name = local.resource_names.storage_container azure_location = var.bootstrap_location + managed_identity_ids = module.identities.managed_identity_ids + managed_identity_client_ids = module.identities.managed_identity_client_ids + managed_identity_principal_ids = module.identities.managed_identity_principal_ids target_subscriptions = local.target_subscriptions root_parent_management_group_id = local.root_parent_management_group_id agent_container_instances = local.runner_container_instances @@ -89,7 +106,7 @@ module "github" { repository_files = local.repository_files template_repository_files = local.template_repository_files workflows = local.workflows - managed_identity_client_ids = module.azure.user_assigned_managed_identity_client_ids + managed_identity_client_ids = module.identities.managed_identity_client_ids azure_tenant_id = data.azurerm_client_config.current.tenant_id azure_subscription_id = data.azurerm_client_config.current.subscription_id backend_azure_resource_group_name = local.resource_names.resource_group_state @@ -105,3 +122,27 @@ module "github" { use_self_hosted_runners = var.use_self_hosted_runners create_branch_policies = var.create_branch_policies } + +# ======================================== +# STATE MIGRATION (Backwards Compatibility) +# ======================================== +# These moved blocks ensure existing deployments migrate smoothly to the new identity module structure + +moved { + from = module.azure.azurerm_resource_group.identity + to = module.identities.azurerm_resource_group.identity +} + +moved { + from = module.azure.azurerm_user_assigned_identity.alz["plan"] + to = module.identities.azurerm_user_assigned_identity.identities["plan"] +} + +moved { + from = module.azure.azurerm_user_assigned_identity.alz["apply"] + to = module.identities.azurerm_user_assigned_identity.identities["apply"] +} + +# Note: Federated credentials in github are dynamically created based on subjects +# The moved blocks need to account for the dynamic keys from module.github.subjects +# Users may need to manually move these or use terraform state mv commands diff --git a/alz/github/variables.tf b/alz/github/variables.tf index 6810568..98ded6b 100644 --- a/alz/github/variables.tf +++ b/alz/github/variables.tf @@ -705,3 +705,10 @@ variable "storage_account_blob_versioning_enabled" { type = bool default = true } + + +variable "tags" { + type = map(string) + default = {} + description = "Tags to apply to Azure resources" +} From b103a77c1e196f5a5a10523ec9a56885e2a7b2f1 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:51:01 +0100 Subject: [PATCH 05/14] feat(local): integrate identities module with conditional creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add module.identities with conditional count parameter - Conditionally create identities based on create_bootstrap_resources_in_azure - Update module.azure to consume identity outputs with [0] indexing - Add moved blocks for backwards-compatible state migration - Add tags variable support for identity resources Local orchestration pattern differs from azuredevops/github due to optional Azure resource creation. Moved blocks ensure existing deployments upgrade without resource recreation: - module.azure[0].azurerm_resource_group.identity → module.identities[0].azurerm_resource_group.identity - module.azure[0].azurerm_user_assigned_identity.alz → module.identities[0].azurerm_user_assigned_identity.identities - module.azure[0].azurerm_federated_identity_credential.alz → module.identities[0].azurerm_federated_identity_credential.credentials --- alz/local/main.tf | 37 ++++++++++++++++++++++++++++++++++--- alz/local/variables.tf | 6 ++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/alz/local/main.tf b/alz/local/main.tf index f032202..8f829e3 100644 --- a/alz/local/main.tf +++ b/alz/local/main.tf @@ -32,12 +32,28 @@ module "files" { additional_folders_path = var.additional_folders_path } +# ======================================== +# IDENTITY MANAGEMENT (Optional for local deployment) +# ======================================== +# Create bootstrap identities only if deploying Azure resources + +module "identities" { + count = var.create_bootstrap_resources_in_azure ? 1 : 0 + source = "../../modules/identities" + resource_group_name = local.resource_names.resource_group_identity + location = var.bootstrap_location + managed_identities = local.managed_identities + federated_credentials = var.federated_credentials + tags = var.tags +} + module "azure" { source = "../../modules/azure" count = var.create_bootstrap_resources_in_azure ? 1 : 0 - user_assigned_managed_identities = local.managed_identities - federated_credentials = local.federated_credentials - resource_group_identity_name = local.resource_names.resource_group_identity + managed_identity_ids = module.identities[0].managed_identity_ids + managed_identity_client_ids = module.identities[0].managed_identity_client_ids + managed_identity_principal_ids = module.identities[0].managed_identity_principal_ids + resource_group_identity_name = module.identities[0].resource_group_name resource_group_state_name = local.resource_names.resource_group_state create_storage_account = var.iac_type == local.iac_terraform storage_account_name = local.resource_names.storage_account @@ -69,3 +85,18 @@ resource "local_file" "command" { content = local.command_final filename = "${local.target_directory}/scripts/deploy-local.ps1" } + +moved { + from = module.azure[0].azurerm_resource_group.identity + to = module.identities[0].azurerm_resource_group.identity +} + +moved { + from = module.azure[0].azurerm_user_assigned_identity.alz + to = module.identities[0].azurerm_user_assigned_identity.identities +} + +moved { + from = module.azure[0].azurerm_federated_identity_credential.alz + to = module.identities[0].azurerm_federated_identity_credential.credentials +} diff --git a/alz/local/variables.tf b/alz/local/variables.tf index e31db50..7c7d627 100644 --- a/alz/local/variables.tf +++ b/alz/local/variables.tf @@ -511,3 +511,9 @@ variable "storage_account_blob_versioning_enabled" { type = bool default = true } + +variable "tags" { + type = map(string) + description = "(Optional) Tags to apply to resources." + default = {} +} From d30845b6c66680d229078b664f9632bf8b651f00 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:26:30 +0100 Subject: [PATCH 06/14] fix(azure): update storage.tf role assignments to use managed_identity_principal_ids - Replace var.user_assigned_managed_identities with var.managed_identity_principal_ids - Replace azurerm_user_assigned_identity.alz[each.key].principal_id with each.value - Fixes role assignments for storage container and storage reader roles - Completes azure module refactoring for identity consumption --- modules/azure/storage.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/azure/storage.tf b/modules/azure/storage.tf index e606750..8c8503c 100644 --- a/modules/azure/storage.tf +++ b/modules/azure/storage.tf @@ -59,10 +59,10 @@ resource "azapi_resource" "storage_account_container" { } resource "azurerm_role_assignment" "alz_storage_container" { - for_each = var.create_storage_account ? var.user_assigned_managed_identities : {} + for_each = var.create_storage_account ? var.managed_identity_principal_ids : {} scope = azapi_resource.storage_account_container[0].id role_definition_name = "Storage Blob Data Owner" - principal_id = azurerm_user_assigned_identity.alz[each.key].principal_id + principal_id = each.value } resource "azurerm_role_assignment" "alz_storage_container_additional" { @@ -75,10 +75,10 @@ resource "azurerm_role_assignment" "alz_storage_container_additional" { # These role assignments are a temporary addition to handle this issue in the Terraform CLI: https://github.com/hashicorp/terraform/issues/36595 # They will be removed once the issue has been resolved resource "azurerm_role_assignment" "alz_storage_reader" { - for_each = var.create_storage_account ? var.user_assigned_managed_identities : {} + for_each = var.create_storage_account ? var.managed_identity_principal_ids : {} scope = azurerm_storage_account.alz[0].id role_definition_name = "Reader" - principal_id = azurerm_user_assigned_identity.alz[each.key].principal_id + principal_id = each.value } resource "azurerm_role_assignment" "alz_storage_reader_additional" { From 7f8f18736730645569ac54861f2e5abeaea462ed Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:45:09 +0100 Subject: [PATCH 07/14] fix: rename default Azure DevOps repository when project name matches repo name to avoid conflicts --- modules/azure_devops/repository_module.tf | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/azure_devops/repository_module.tf b/modules/azure_devops/repository_module.tf index a1f960d..9276177 100644 --- a/modules/azure_devops/repository_module.tf +++ b/modules/azure_devops/repository_module.tf @@ -1,11 +1,31 @@ +# When creating a new project, Azure DevOps automatically creates a default repository +# with the same name as the project. If our repository name matches the project name, +# we need to rename the default repo first to avoid conflicts. +resource "azuredevops_git_repository" "default_rename" { + count = var.create_project && var.repository_name == var.project_name ? 1 : 0 + project_id = local.project_id + name = "${var.project_name}-default" + default_branch = "refs/heads/main" + + lifecycle { + ignore_changes = [initialization, default_branch] + } + + depends_on = [azuredevops_project.alz] +} + resource "azuredevops_git_repository" "alz" { - depends_on = [azuredevops_environment.alz] + depends_on = [azuredevops_environment.alz, azuredevops_git_repository.default_rename] project_id = local.project_id name = var.repository_name default_branch = local.default_branch initialization { init_type = "Clean" } + + lifecycle { + ignore_changes = [initialization] + } } resource "azuredevops_git_repository_file" "alz" { From 50c8bbf9a27d3fbdc96efdaac4e8e016611cd705 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:50:31 +0100 Subject: [PATCH 08/14] fix: correct repository_module.tf syntax errors --- modules/azure_devops/repository_module.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/azure_devops/repository_module.tf b/modules/azure_devops/repository_module.tf index 9276177..5896fc3 100644 --- a/modules/azure_devops/repository_module.tf +++ b/modules/azure_devops/repository_module.tf @@ -7,6 +7,10 @@ resource "azuredevops_git_repository" "default_rename" { name = "${var.project_name}-default" default_branch = "refs/heads/main" + initialization { + init_type = "Clean" + } + lifecycle { ignore_changes = [initialization, default_branch] } @@ -19,6 +23,7 @@ resource "azuredevops_git_repository" "alz" { project_id = local.project_id name = var.repository_name default_branch = local.default_branch + initialization { init_type = "Clean" } From b09452113743cdcc97f1bfc20387a73372c5f496 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:56:49 +0100 Subject: [PATCH 09/14] fix: remove count dependency on unknown values in default_rename resource --- modules/azure_devops/repository_module.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/azure_devops/repository_module.tf b/modules/azure_devops/repository_module.tf index 5896fc3..01eb3e6 100644 --- a/modules/azure_devops/repository_module.tf +++ b/modules/azure_devops/repository_module.tf @@ -1,9 +1,9 @@ # When creating a new project, Azure DevOps automatically creates a default repository -# with the same name as the project. If our repository name matches the project name, -# we need to rename the default repo first to avoid conflicts. +# with the same name as the project. We need to rename this default repo to avoid conflicts +# with our Terraform-managed repository. resource "azuredevops_git_repository" "default_rename" { - count = var.create_project && var.repository_name == var.project_name ? 1 : 0 - project_id = local.project_id + count = var.create_project ? 1 : 0 + project_id = azuredevops_project.alz[0].id name = "${var.project_name}-default" default_branch = "refs/heads/main" From 06e612ce35e1367ebc93b5ed6182fe5703479663 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Thu, 27 Nov 2025 14:35:56 +0100 Subject: [PATCH 10/14] feat: add Container App Jobs support for Azure DevOps agents This commit adds support for Azure Container App Jobs as an alternative to Container Instances for running self-hosted Azure DevOps agents. Changes: - Add container_app_jobs module integration with AVM pattern module v0.5 - Add use_container_app_jobs variable to enable/disable Container App Jobs - Add Container App subnet configuration (10.0.4.0/23) - Add agent_container_cpu and agent_container_memory variables (defaults: 2 cores, 4Gi) - Add managed identity support for agent authentication (no PAT token required) - Add Container App Environment with Log Analytics workspace integration - Update resource dependencies and role assignments for BYO mode Fixes in this amendment: - Remove duplicate agent_pool_name variable declaration (line 274) - Correct AVM module parameters to match v0.5.0 interface - Add required parameters: location, postfix, version_control_system_organization, version_control_system_type - Add compute_types for azure_container_app - Use flat identity structure (user_assigned_managed_identity_id/client_id/principal_id) - Use correct BYO parameters (virtual_network_name, container_registry_name) - Add Container App configuration (cpu, memory, execution counts, polling interval) - Use UAMI authentication with null PAT token - Add random_string resource for postfix requirement Co-authored-by: GitHub Copilot --- README.md | 8 +++ alz/azuredevops/agent_identity_permissions.tf | 16 +++++ alz/azuredevops/locals.tf | 16 +++-- alz/azuredevops/main.tf | 5 ++ alz/azuredevops/variables.tf | 19 ++++++ alz/github/main.tf | 5 ++ alz/github/variables.tf | 6 ++ modules/azure/container_instances.tf | 4 +- modules/azure/container_registry.tf | 16 ++++- modules/azure/main.container.app.jobs.tf | 63 +++++++++++++++++++ modules/azure/networking.tf | 21 +++++++ modules/azure/variables.tf | 30 +++++++++ 12 files changed, 201 insertions(+), 8 deletions(-) create mode 100644 alz/azuredevops/agent_identity_permissions.tf create mode 100644 modules/azure/main.container.app.jobs.tf diff --git a/README.md b/README.md index 758d3bd..589bc95 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,11 @@ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/Azure/accelerator-bootstrap-modules/badge)](https://scorecard.dev/viewer/?uri=github.com/Azure/accelerator-bootstrap-modules) This repository contains the Terraform modules that are used to deploy the accelerator bootstrap environments. + +## Features + +- Azure DevOps and GitHub bootstrap orchestrations +- User-assigned managed identities with federated credentials +- Self-hosted agents with Container Instances or Container App Jobs (Azure DevOps only) +- Private networking support with NAT Gateway +- Container Registry with ACR Tasks for agent images diff --git a/alz/azuredevops/agent_identity_permissions.tf b/alz/azuredevops/agent_identity_permissions.tf new file mode 100644 index 0000000..79ad807 --- /dev/null +++ b/alz/azuredevops/agent_identity_permissions.tf @@ -0,0 +1,16 @@ +# Azure DevOps group membership for agent identity +# The agent identity must be a member of "Project Collection Service Accounts" group +# to access Azure DevOps resources with managed identity authentication + +data "azuredevops_group" "project_collection_service_accounts" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + name = "Project Collection Service Accounts" +} + +resource "azuredevops_group_membership" "agent_identity" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + group = data.azuredevops_group.project_collection_service_accounts[0].descriptor + members = [ + module.identities.managed_identity_principal_ids["agent"] + ] +} diff --git a/alz/azuredevops/locals.tf b/alz/azuredevops/locals.tf index 2448619..9a8fe5b 100644 --- a/alz/azuredevops/locals.tf +++ b/alz/azuredevops/locals.tf @@ -12,7 +12,8 @@ locals { } locals { - use_private_networking = var.use_self_hosted_agents && var.use_private_networking + # Auto-enable private networking when using Container App Jobs (Container App Environment requires VNET) + use_private_networking = var.use_self_hosted_agents && (var.use_private_networking || var.use_container_app_jobs) allow_storage_access_from_my_ip = local.use_private_networking && var.allow_storage_access_from_my_ip } @@ -34,10 +35,15 @@ locals { } locals { - managed_identities = { - (local.plan_key) = local.resource_names.user_assigned_managed_identity_plan - (local.apply_key) = local.resource_names.user_assigned_managed_identity_apply - } + managed_identities = merge( + { + (local.plan_key) = local.resource_names.user_assigned_managed_identity_plan + (local.apply_key) = local.resource_names.user_assigned_managed_identity_apply + }, + var.use_self_hosted_agents && var.use_container_app_jobs ? { + agent = local.resource_names.user_assigned_managed_identity_agent + } : {} + ) # Federated credentials for the identities module (includes audience) federated_credentials_for_identities = { diff --git a/alz/azuredevops/main.tf b/alz/azuredevops/main.tf index 284e973..224d763 100644 --- a/alz/azuredevops/main.tf +++ b/alz/azuredevops/main.tf @@ -69,16 +69,21 @@ module "azure" { virtual_network_name = local.resource_names.virtual_network virtual_network_subnet_name_container_instances = local.resource_names.subnet_container_instances virtual_network_subnet_name_private_endpoints = local.resource_names.subnet_private_endpoints + virtual_network_subnet_name_container_apps = var.use_container_app_jobs ? local.resource_names.subnet_container_apps : "" storage_account_private_endpoint_name = local.resource_names.storage_account_private_endpoint use_private_networking = local.use_private_networking allow_storage_access_from_my_ip = local.allow_storage_access_from_my_ip virtual_network_address_space = var.virtual_network_address_space virtual_network_subnet_address_prefix_container_instances = var.virtual_network_subnet_address_prefix_container_instances virtual_network_subnet_address_prefix_private_endpoints = var.virtual_network_subnet_address_prefix_private_endpoints + virtual_network_subnet_address_prefix_container_apps = var.virtual_network_subnet_address_prefix_container_apps storage_account_replication_type = var.storage_account_replication_type public_ip_name = local.resource_names.public_ip nat_gateway_name = local.resource_names.nat_gateway use_self_hosted_agents = var.use_self_hosted_agents + use_container_app_jobs = var.use_container_app_jobs + agent_container_cpu = var.agent_container_cpu + agent_container_memory = var.agent_container_memory container_registry_name = local.resource_names.container_registry container_registry_private_endpoint_name = local.resource_names.container_registry_private_endpoint container_registry_image_name = local.resource_names.container_image_name diff --git a/alz/azuredevops/variables.tf b/alz/azuredevops/variables.tf index d0329f8..6f3c727 100644 --- a/alz/azuredevops/variables.tf +++ b/alz/azuredevops/variables.tf @@ -164,6 +164,17 @@ variable "use_self_hosted_agents" { default = true } +variable "use_container_app_jobs" { + description = "Whether to use Container App Jobs for self-hosted agents (Azure DevOps only). Mutually exclusive with Container Instances." + type = bool + default = false + + validation { + condition = !var.use_container_app_jobs || var.use_self_hosted_agents + error_message = "use_container_app_jobs requires use_self_hosted_agents to be true." + } +} + variable "azure_devops_agents_personal_access_token" { description = "Personal access token for Azure DevOps self-hosted agents (the token requires the 'Agent Pools - Read & Manage' scope and should have the maximum expiry). Only required if 'use_self_hosted_runners' is 'true'" type = string @@ -281,6 +292,7 @@ variable "resource_names" { resource_group_network = optional(string, "rg-{{service_name}}-{{environment_name}}-network-{{azure_location}}-{{postfix_number}}") user_assigned_managed_identity_plan = optional(string, "id-{{service_name}}-{{environment_name}}-{{azure_location}}-plan-{{postfix_number}}") user_assigned_managed_identity_apply = optional(string, "id-{{service_name}}-{{environment_name}}-{{azure_location}}-apply-{{postfix_number}}") + user_assigned_managed_identity_agent = optional(string, "id-{{service_name}}-{{environment_name}}-{{azure_location}}-agent-{{postfix_number}}") user_assigned_managed_identity_federated_credentials_plan = optional(string, "id-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-plan") user_assigned_managed_identity_federated_credentials_apply = optional(string, "id-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-apply") storage_account = optional(string, "sto{{service_name_short}}{{environment_name_short}}{{azure_location_short}}{{postfix_number}}{{random_string}}") @@ -306,6 +318,7 @@ variable "resource_names" { nat_gateway = optional(string, "nat-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}") subnet_container_instances = optional(string, "subnet-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-aci") subnet_private_endpoints = optional(string, "subnet-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-pe") + subnet_container_apps = optional(string, "subnet-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-ca") storage_account_private_endpoint = optional(string, "pe-{{service_name}}-{{environment_name}}-{{azure_location}}-sto-{{postfix_number}}") container_registry = optional(string, "acr{{service_name}}{{environment_name}}{{azure_location_short}}{{postfix_number}}{{random_string}}") container_registry_private_endpoint = optional(string, "pe-{{service_name}}-{{environment_name}}-{{azure_location}}-acr-{{postfix_number}}") @@ -357,6 +370,12 @@ variable "virtual_network_subnet_address_prefix_private_endpoints" { default = "10.0.0.64/26" } +variable "virtual_network_subnet_address_prefix_container_apps" { + type = string + description = "Address prefix for the Container Apps subnet" + default = "10.0.0.128/26" +} + variable "storage_account_replication_type" { description = "Controls the redundancy for the storage account" type = string diff --git a/alz/github/main.tf b/alz/github/main.tf index e7dde9e..b239c0a 100644 --- a/alz/github/main.tf +++ b/alz/github/main.tf @@ -1,3 +1,8 @@ +# Validation: Container App Jobs are not supported for GitHub +locals { + validation_container_app_jobs_not_supported = var.use_self_hosted_agents && var.use_container_app_jobs ? tobool("ERROR: Container App Jobs (use_container_app_jobs=true) are only supported with Azure DevOps, not GitHub. Please use Container Instances instead (use_container_app_jobs=false).") : true +} + module "resource_names" { source = "../../modules/resource_names" azure_location = var.bootstrap_location diff --git a/alz/github/variables.tf b/alz/github/variables.tf index 98ded6b..d800a6b 100644 --- a/alz/github/variables.tf +++ b/alz/github/variables.tf @@ -181,6 +181,12 @@ variable "use_self_hosted_runners" { default = true } +variable "use_container_app_jobs" { + description = "NOT SUPPORTED FOR GITHUB. Container App Jobs are only supported with Azure DevOps. This variable exists for validation purposes only." + type = bool + default = false +} + variable "github_runners_personal_access_token" { description = "Personal access token for GitHub self-hosted runners (the token requires the 'repo' scope and should not expire). Only required if 'use_self_hosted_runners' is 'true'" type = string diff --git a/modules/azure/container_instances.tf b/modules/azure/container_instances.tf index 0c2b073..59dd919 100644 --- a/modules/azure/container_instances.tf +++ b/modules/azure/container_instances.tf @@ -1,5 +1,5 @@ resource "azurerm_container_group" "alz" { - for_each = var.use_self_hosted_agents ? var.agent_container_instances : {} + for_each = var.use_self_hosted_agents && !var.use_container_app_jobs ? var.agent_container_instances : {} name = each.value.container_instance_name location = var.azure_location resource_group_name = azurerm_resource_group.agents[0].name @@ -49,7 +49,7 @@ resource "azurerm_container_group" "alz" { } resource "azurerm_user_assigned_identity" "container_instances" { - count = var.use_self_hosted_agents ? 1 : 0 + count = var.use_self_hosted_agents && !var.use_container_app_jobs ? 1 : 0 location = var.azure_location name = var.agent_container_instance_managed_identity_name resource_group_name = azurerm_resource_group.agents[0].name diff --git a/modules/azure/container_registry.tf b/modules/azure/container_registry.tf index 5b1c40c..b7e9408 100644 --- a/modules/azure/container_registry.tf +++ b/modules/azure/container_registry.tf @@ -57,7 +57,7 @@ resource "azurerm_container_registry_task_schedule_run_now" "alz" { } resource "azurerm_role_assignment" "container_registry_pull_for_container_instance" { - count = var.use_self_hosted_agents ? 1 : 0 + count = var.use_self_hosted_agents && !var.use_container_app_jobs ? 1 : 0 scope = azurerm_container_registry.alz[0].id role_definition_name = "AcrPull" principal_id = azurerm_user_assigned_identity.container_instances[0].principal_id @@ -69,3 +69,17 @@ resource "azurerm_role_assignment" "container_registry_push_for_task" { role_definition_name = "AcrPush" principal_id = azurerm_container_registry_task.alz[0].identity[0].principal_id } + +resource "azurerm_role_assignment" "container_registry_pull_for_agent" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + scope = azurerm_container_registry.alz[0].id + role_definition_name = "AcrPull" + principal_id = var.managed_identity_principal_ids["agent"] +} + +resource "azurerm_role_assignment" "container_registry_push_for_agent" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + scope = azurerm_container_registry.alz[0].id + role_definition_name = "AcrPush" + principal_id = var.managed_identity_principal_ids["agent"] +} diff --git a/modules/azure/main.container.app.jobs.tf b/modules/azure/main.container.app.jobs.tf new file mode 100644 index 0000000..cf98812 --- /dev/null +++ b/modules/azure/main.container.app.jobs.tf @@ -0,0 +1,63 @@ +module "container_app_jobs" { + source = "Azure/avm-ptn-cicd-agents-and-runners/azurerm" + version = "~> 0.5" + + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + + # Required inputs + location = var.azure_location + postfix = random_string.container_app_postfix[0].result + version_control_system_organization = var.agent_organization_url + version_control_system_type = "azuredevops" + + # Compute type + compute_types = ["azure_container_app"] + + # Resource group (BYO mode) + resource_group_creation_enabled = false + resource_group_name = azurerm_resource_group.agents[0].name + + # Virtual network (BYO mode) + virtual_network_creation_enabled = false + virtual_network_name = azurerm_virtual_network.alz[0].name + virtual_network_address_space = var.virtual_network_address_space + + # Container registry (BYO mode) + container_registry_creation_enabled = false + container_registry_name = azurerm_container_registry.alz[0].name + + # User-assigned managed identity (BYO mode) + user_assigned_managed_identity_creation_enabled = false + user_assigned_managed_identity_id = var.managed_identity_ids["agent"] + user_assigned_managed_identity_client_id = var.managed_identity_client_ids["agent"] + user_assigned_managed_identity_principal_id = var.managed_identity_principal_ids["agent"] + + # Container App Job configuration + container_app_container_cpu = var.agent_container_cpu + container_app_container_memory = "${var.agent_container_memory}Gi" + container_app_max_execution_count = 10 + container_app_min_execution_count = 0 + container_app_polling_interval_seconds = 30 + + # Version control system configuration + version_control_system_authentication_method = "uami" + version_control_system_personal_access_token = null + version_control_system_pool_name = var.agent_pool_name + + # Private networking + use_private_networking = true + + depends_on = [ + azurerm_role_assignment.container_registry_pull_for_agent, + azurerm_role_assignment.container_registry_push_for_agent + ] +} + +# Random string for postfix (AVM module requires max 20 chars) +resource "random_string" "container_app_postfix" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + length = 6 + special = false + upper = false + numeric = true +} diff --git a/modules/azure/networking.tf b/modules/azure/networking.tf index d2b5d10..3c130f9 100644 --- a/modules/azure/networking.tf +++ b/modules/azure/networking.tf @@ -59,3 +59,24 @@ resource "azurerm_subnet" "private_endpoints" { address_prefixes = [var.virtual_network_subnet_address_prefix_private_endpoints] private_endpoint_network_policies = "Enabled" } + +resource "azurerm_subnet" "container_apps" { + count = var.use_private_networking && var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + name = var.virtual_network_subnet_name_container_apps + resource_group_name = azurerm_resource_group.network[0].name + virtual_network_name = azurerm_virtual_network.alz[0].name + address_prefixes = [var.virtual_network_subnet_address_prefix_container_apps] + delegation { + name = "container-app-delegation" + service_delegation { + name = "Microsoft.App/environments" + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + } + } +} + +resource "azurerm_subnet_nat_gateway_association" "container_apps" { + count = var.use_private_networking && var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + subnet_id = azurerm_subnet.container_apps[0].id + nat_gateway_id = azurerm_nat_gateway.alz[0].id +} diff --git a/modules/azure/variables.tf b/modules/azure/variables.tf index 9b45526..d8c4d5c 100644 --- a/modules/azure/variables.tf +++ b/modules/azure/variables.tf @@ -197,6 +197,18 @@ variable "virtual_network_subnet_address_prefix_private_endpoints" { default = "10.0.0.64/26" } +variable "virtual_network_subnet_name_container_apps" { + type = string + description = "Name of the virtual network subnet for Container Apps" + default = "" +} + +variable "virtual_network_subnet_address_prefix_container_apps" { + type = string + description = "Address prefix for the Container Apps subnet" + default = "10.0.0.128/26" +} + variable "storage_account_private_endpoint_name" { type = string default = "" @@ -253,6 +265,24 @@ variable "use_self_hosted_agents" { default = true } +variable "use_container_app_jobs" { + type = bool + default = false + description = "Whether to use Container App Jobs for self-hosted agents (Azure DevOps only). Mutually exclusive with Container Instances." +} + +variable "agent_container_cpu" { + type = number + default = 2 + description = "CPU allocation for agent containers" +} + +variable "agent_container_memory" { + type = string + default = "4Gi" + description = "Memory allocation for agent containers" +} + variable "agent_container_instance_managed_identity_name" { type = string default = "" From ac8e321331215a42acd4013076083fb7e3288184 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Thu, 27 Nov 2025 15:20:57 +0100 Subject: [PATCH 11/14] fix: add missing virtual_network_id and container_app_subnet_id parameters - Add virtual_network_id parameter required for private DNS zone link - Use container_app_subnet_id instead of incorrectly named virtual_network_subnet_id - Required for Container App Environment with private networking --- modules/azure/main.container.app.jobs.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/azure/main.container.app.jobs.tf b/modules/azure/main.container.app.jobs.tf index cf98812..a48dd9b 100644 --- a/modules/azure/main.container.app.jobs.tf +++ b/modules/azure/main.container.app.jobs.tf @@ -21,6 +21,8 @@ module "container_app_jobs" { virtual_network_creation_enabled = false virtual_network_name = azurerm_virtual_network.alz[0].name virtual_network_address_space = var.virtual_network_address_space + virtual_network_id = azurerm_virtual_network.alz[0].id + container_app_subnet_id = azurerm_subnet.container_apps[0].id # Container registry (BYO mode) container_registry_creation_enabled = false From cb6fdfffd5a7f696155f942cfa3ad29349869f23 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Thu, 27 Nov 2025 15:27:42 +0100 Subject: [PATCH 12/14] fix: correct container registry parameters for BYO mode - Use custom_container_registry_login_server instead of container_registry_name - Add container_registry_private_endpoint_subnet_id for private networking - Enable container_registry_private_dns_zone_creation_enabled - Remove double 'Gi' suffix from container_app_container_memory (already in variable) - Fixes null registry_login_server error in container-app-job module --- modules/azure/main.container.app.jobs.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/azure/main.container.app.jobs.tf b/modules/azure/main.container.app.jobs.tf index a48dd9b..06ae5da 100644 --- a/modules/azure/main.container.app.jobs.tf +++ b/modules/azure/main.container.app.jobs.tf @@ -26,7 +26,13 @@ module "container_app_jobs" { # Container registry (BYO mode) container_registry_creation_enabled = false - container_registry_name = azurerm_container_registry.alz[0].name + custom_container_registry_login_server = azurerm_container_registry.alz[0].login_server + + # Container registry private endpoint subnet (BYO mode) + container_registry_private_endpoint_subnet_id = azurerm_subnet.private_endpoints[0].id + + # Container registry DNS zone (BYO mode - let module create) + container_registry_private_dns_zone_creation_enabled = true # User-assigned managed identity (BYO mode) user_assigned_managed_identity_creation_enabled = false @@ -36,7 +42,7 @@ module "container_app_jobs" { # Container App Job configuration container_app_container_cpu = var.agent_container_cpu - container_app_container_memory = "${var.agent_container_memory}Gi" + container_app_container_memory = var.agent_container_memory container_app_max_execution_count = 10 container_app_min_execution_count = 0 container_app_polling_interval_seconds = 30 From e9e226e0c6ec498feb0a253c51050cd2d817acb4 Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Thu, 27 Nov 2025 16:01:22 +0100 Subject: [PATCH 13/14] fix: add UAMI Azure DevOps service principal entitlement and fix DNS zone duplicate - Add time_sleep resource for 30s UAMI propagation delay - Add azuredevops_service_principal_entitlement to register UAMI as service principal - Change group membership to use entitlement descriptor instead of principal_id - Fix duplicate DNS zone link by using existing container_registry DNS zone - Set container_registry_private_dns_zone_creation_enabled = false Fixes two deployment errors: 1. UAMI not registered in Azure DevOps (controller not found error) 2. Duplicate private DNS zone virtual network link error Pattern based on AVM module example: https://github.com/Azure/terraform-azurerm-avm-ptn-cicd-agents-and-runners/blob/main/examples/azure_devops_container_app_uami/main.tf --- alz/azuredevops/agent_identity_permissions.tf | 37 +++++++++++++++---- modules/azure/main.container.app.jobs.tf | 13 +++---- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/alz/azuredevops/agent_identity_permissions.tf b/alz/azuredevops/agent_identity_permissions.tf index 79ad807..0044ec2 100644 --- a/alz/azuredevops/agent_identity_permissions.tf +++ b/alz/azuredevops/agent_identity_permissions.tf @@ -1,16 +1,39 @@ -# Azure DevOps group membership for agent identity -# The agent identity must be a member of "Project Collection Service Accounts" group -# to access Azure DevOps resources with managed identity authentication +# Azure DevOps Service Principal Entitlement for Agent Identity +# Required for UAMI authentication to work with Azure DevOps +# Pattern from: https://github.com/Azure/terraform-azurerm-avm-ptn-cicd-agents-and-runners/blob/main/examples/azure_devops_container_app_uami/main.tf data "azuredevops_group" "project_collection_service_accounts" { count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 name = "Project Collection Service Accounts" } +# Wait for UAMI propagation in Azure AD +resource "time_sleep" "agent_identity_propagation" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + create_duration = "30s" + + depends_on = [module.identities] +} + +# Create service principal entitlement for the agent UAMI +resource "azuredevops_service_principal_entitlement" "agent_identity" { + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + account_license_type = "express" # Basic license for service principals + origin = "aad" # Azure Active Directory + origin_id = module.identities.managed_identity_principal_ids["agent"] + + depends_on = [time_sleep.agent_identity_propagation] +} + +# Add agent identity service principal to Project Collection Service Accounts group resource "azuredevops_group_membership" "agent_identity" { - count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 - group = data.azuredevops_group.project_collection_service_accounts[0].descriptor - members = [ - module.identities.managed_identity_principal_ids["agent"] + count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 + group = data.azuredevops_group.project_collection_service_accounts[0].descriptor + members = [azuredevops_service_principal_entitlement.agent_identity[0].descriptor] + mode = "add" + + depends_on = [ + azuredevops_service_principal_entitlement.agent_identity, + data.azuredevops_group.project_collection_service_accounts ] } diff --git a/modules/azure/main.container.app.jobs.tf b/modules/azure/main.container.app.jobs.tf index 06ae5da..59c93bb 100644 --- a/modules/azure/main.container.app.jobs.tf +++ b/modules/azure/main.container.app.jobs.tf @@ -25,14 +25,11 @@ module "container_app_jobs" { container_app_subnet_id = azurerm_subnet.container_apps[0].id # Container registry (BYO mode) - container_registry_creation_enabled = false - custom_container_registry_login_server = azurerm_container_registry.alz[0].login_server - - # Container registry private endpoint subnet (BYO mode) - container_registry_private_endpoint_subnet_id = azurerm_subnet.private_endpoints[0].id - - # Container registry DNS zone (BYO mode - let module create) - container_registry_private_dns_zone_creation_enabled = true + container_registry_creation_enabled = false + custom_container_registry_login_server = azurerm_container_registry.alz[0].login_server + container_registry_private_endpoint_subnet_id = azurerm_subnet.private_endpoints[0].id + container_registry_dns_zone_id = azurerm_private_dns_zone.alz["container_registry"].id + container_registry_private_dns_zone_creation_enabled = false # User-assigned managed identity (BYO mode) user_assigned_managed_identity_creation_enabled = false From 46163c0a9f3f1738897ad76dc766b6a8912087cf Mon Sep 17 00:00:00 2001 From: Haflidi Fridthjofsson <26624010+haflidif@users.noreply.github.com> Date: Tue, 2 Dec 2025 18:12:52 +0100 Subject: [PATCH 14/14] feat: add Azure Container App Jobs support for self-hosted agents Add support for deploying Azure DevOps agents using Azure Container App Jobs as an alternative to Azure Container Instances. Container App Jobs provide event-driven scaling with KEDA and better integration with Container Apps infrastructure. Key features: - BYO (Bring Your Own) mode integration with existing infrastructure - Custom container image support with configurable repository and tags - Managed identity authentication for ACR access - Private networking support with VNet integration - Structured naming convention following bootstrap patterns Changes: - Add container_app_jobs.tf module for Container App Jobs deployment - Integrate with AVM pattern module (avm-ptn-cicd-agents-and-runners v0.5) - Add variables for Container App naming (environment, job, placeholder) - Configure memory formatting (4 -> 4Gi) for Container Apps API - Add service_name and environment_name variables for naming - Add separate image configuration for ACA vs ACI deployments - Use location abbreviation for Container App Jobs naming (32 char limit) Resources created: - Container App Environment (cae-*) - Container App Job (caj-*) - Container App Job Placeholder (caj-*-ph) - Infrastructure Resource Group (rg-*-ca-infra) New variables: - agent_container_app_image_tag: Image tag for Container App Jobs (default: 221742d) - agent_container_app_image_folder: Dockerfile folder for ACA (default: azure-devops-agent-aca) - container_app_environment_name: Name for Container App Environment - container_app_job_name: Name for Container App Job - container_app_job_placeholder_name: Name for placeholder job - container_app_infrastructure_resource_group_name: Name for infra RG --- alz/azuredevops/locals.tf | 6 ++- alz/azuredevops/main.tf | 8 +++- alz/azuredevops/variables.tf | 25 ++++++++++-- ...iner.app.jobs.tf => container_app_jobs.tf} | 31 +++++++++------ modules/azure/variables.tf | 39 ++++++++++++++++++- 5 files changed, 91 insertions(+), 18 deletions(-) rename modules/azure/{main.container.app.jobs.tf => container_app_jobs.tf} (69%) diff --git a/alz/azuredevops/locals.tf b/alz/azuredevops/locals.tf index 9a8fe5b..33a20f3 100644 --- a/alz/azuredevops/locals.tf +++ b/alz/azuredevops/locals.tf @@ -111,7 +111,11 @@ locals { } locals { - agent_container_instance_dockerfile_url = "${var.agent_container_image_repository}#${var.agent_container_image_tag}:${var.agent_container_image_folder}" + # Use different image folder and tag for Container App Jobs vs Container Instances + agent_image_folder = var.use_container_app_jobs ? var.agent_container_app_image_folder : var.agent_container_image_folder + agent_image_tag = var.use_container_app_jobs ? var.agent_container_app_image_tag : var.agent_container_image_tag + + agent_container_instance_dockerfile_url = "${var.agent_container_image_repository}#${local.agent_image_tag}:${local.agent_image_folder}" } locals { diff --git a/alz/azuredevops/main.tf b/alz/azuredevops/main.tf index 224d763..3ec27d3 100644 --- a/alz/azuredevops/main.tf +++ b/alz/azuredevops/main.tf @@ -84,10 +84,16 @@ module "azure" { use_container_app_jobs = var.use_container_app_jobs agent_container_cpu = var.agent_container_cpu agent_container_memory = var.agent_container_memory + service_name = var.service_name + environment_name = var.environment_name + container_app_environment_name = local.resource_names.container_app_environment + container_app_job_name = local.resource_names.container_app_job + container_app_job_placeholder_name = local.resource_names.container_app_job_placeholder + container_app_infrastructure_resource_group_name = local.resource_names.container_app_infrastructure_resource_group container_registry_name = local.resource_names.container_registry container_registry_private_endpoint_name = local.resource_names.container_registry_private_endpoint container_registry_image_name = local.resource_names.container_image_name - container_registry_image_tag = var.agent_container_image_tag + container_registry_image_tag = local.agent_image_tag container_registry_dockerfile_name = var.agent_container_image_dockerfile container_registry_dockerfile_repository_folder_url = local.agent_container_instance_dockerfile_url custom_role_definitions = var.iac_type == "terraform" ? local.custom_role_definitions_terraform : local.custom_role_definitions_bicep diff --git a/alz/azuredevops/variables.tf b/alz/azuredevops/variables.tf index 6f3c727..45c3e80 100644 --- a/alz/azuredevops/variables.tf +++ b/alz/azuredevops/variables.tf @@ -168,7 +168,7 @@ variable "use_container_app_jobs" { description = "Whether to use Container App Jobs for self-hosted agents (Azure DevOps only). Mutually exclusive with Container Instances." type = bool default = false - + validation { condition = !var.use_container_app_jobs || var.use_self_hosted_agents error_message = "use_container_app_jobs requires use_self_hosted_agents to be true." @@ -224,18 +224,32 @@ variable "agent_container_image_repository" { default = "https://github.com/Azure/avm-container-images-cicd-agents-and-runners" } +# Container Instances (ACI) configuration variable "agent_container_image_tag" { - description = "The container image tag to use for Azure DevOps Agents" + description = "The container image tag to use for Azure DevOps Agents with Container Instances" type = string default = "39b9059" } variable "agent_container_image_folder" { - description = "The folder containing the Dockerfile for the container image" + description = "The folder containing the Dockerfile for Container Instances" type = string default = "azure-devops-agent-aci" } +# Container App Jobs (ACA) configuration +variable "agent_container_app_image_tag" { + description = "The container image tag to use for Azure DevOps Agents with Container App Jobs" + type = string + default = "221742d" +} + +variable "agent_container_app_image_folder" { + description = "The folder containing the Dockerfile for Container App Jobs" + type = string + default = "azure-devops-agent-aca" +} + variable "agent_container_image_dockerfile" { description = "The Dockerfile to use for the container image" type = string @@ -323,6 +337,11 @@ variable "resource_names" { container_registry = optional(string, "acr{{service_name}}{{environment_name}}{{azure_location_short}}{{postfix_number}}{{random_string}}") container_registry_private_endpoint = optional(string, "pe-{{service_name}}-{{environment_name}}-{{azure_location}}-acr-{{postfix_number}}") container_image_name = optional(string, "azure-devops-agent") + # Container App Jobs naming (max 32 chars) + container_app_environment = optional(string, "cae-{{service_name}}-{{environment_name}}-{{azure_location_short}}-{{postfix_number}}") + container_app_job = optional(string, "caj-{{service_name}}-{{environment_name}}-{{azure_location_short}}-{{postfix_number}}") + container_app_job_placeholder = optional(string, "caj-{{service_name}}-{{environment_name}}-{{azure_location_short}}-{{postfix_number}}-ph") + container_app_infrastructure_resource_group = optional(string, "rg-{{service_name}}-{{environment_name}}-{{azure_location}}-{{postfix_number}}-ca-infra") }) description = "Overrides for resource names" default = {} diff --git a/modules/azure/main.container.app.jobs.tf b/modules/azure/container_app_jobs.tf similarity index 69% rename from modules/azure/main.container.app.jobs.tf rename to modules/azure/container_app_jobs.tf index 59c93bb..75a383c 100644 --- a/modules/azure/main.container.app.jobs.tf +++ b/modules/azure/container_app_jobs.tf @@ -6,10 +6,16 @@ module "container_app_jobs" { # Required inputs location = var.azure_location - postfix = random_string.container_app_postfix[0].result + postfix = "${var.service_name}-${var.environment_name}" version_control_system_organization = var.agent_organization_url version_control_system_type = "azuredevops" + # Override default naming to follow bootstrap naming convention + container_app_environment_name = var.container_app_environment_name + container_app_job_name = var.container_app_job_name + container_app_placeholder_job_name = var.container_app_job_placeholder_name + container_app_infrastructure_resource_group_name = var.container_app_infrastructure_resource_group_name + # Compute type compute_types = ["azure_container_app"] @@ -31,6 +37,18 @@ module "container_app_jobs" { container_registry_dns_zone_id = azurerm_private_dns_zone.alz["container_registry"].id container_registry_private_dns_zone_creation_enabled = false + # Custom container image (use our pre-built image) + use_default_container_image = false + custom_container_registry_images = { + container_app = { + task_name = "image-build-task" + dockerfile_path = var.container_registry_dockerfile_name + context_path = var.container_registry_dockerfile_repository_folder_url + context_access_token = "a" + image_names = ["${var.container_registry_image_name}:${var.container_registry_image_tag}"] + } + } + # User-assigned managed identity (BYO mode) user_assigned_managed_identity_creation_enabled = false user_assigned_managed_identity_id = var.managed_identity_ids["agent"] @@ -39,7 +57,7 @@ module "container_app_jobs" { # Container App Job configuration container_app_container_cpu = var.agent_container_cpu - container_app_container_memory = var.agent_container_memory + container_app_container_memory = "${var.agent_container_memory}Gi" container_app_max_execution_count = 10 container_app_min_execution_count = 0 container_app_polling_interval_seconds = 30 @@ -57,12 +75,3 @@ module "container_app_jobs" { azurerm_role_assignment.container_registry_push_for_agent ] } - -# Random string for postfix (AVM module requires max 20 chars) -resource "random_string" "container_app_postfix" { - count = var.use_self_hosted_agents && var.use_container_app_jobs ? 1 : 0 - length = 6 - special = false - upper = false - numeric = true -} diff --git a/modules/azure/variables.tf b/modules/azure/variables.tf index d8c4d5c..6da291f 100644 --- a/modules/azure/variables.tf +++ b/modules/azure/variables.tf @@ -278,9 +278,44 @@ variable "agent_container_cpu" { } variable "agent_container_memory" { + type = number + default = 4 + description = "Memory allocation for agent containers in Gibibytes" +} + +# Container App Jobs naming variables +variable "container_app_environment_name" { + type = string + default = "" + description = "Name for the Container App Environment" +} + +variable "container_app_job_name" { + type = string + default = "" + description = "Name for the Container App Job" +} + +variable "container_app_job_placeholder_name" { + type = string + default = "" + description = "Name for the Container App Job placeholder" +} + +variable "container_app_infrastructure_resource_group_name" { + type = string + default = "" + description = "Name for the Container Apps infrastructure resource group" +} + +variable "service_name" { + type = string + description = "Service name for resource naming" +} + +variable "environment_name" { type = string - default = "4Gi" - description = "Memory allocation for agent containers" + description = "Environment name for resource naming" } variable "agent_container_instance_managed_identity_name" {