kuberneteswindowssetup.ps1.template

<#
    .SYNOPSIS
        Provisions VM as a Kubernetes agent.

    .DESCRIPTION
        Provisions VM as a Kubernetes agent.

        The parameters passed in are required, and will vary per-deployment.

        Notes on modifying this file:
        - This file extension is PS1, but it is actually used as a template from pkg/engine/template_generator.go
        - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources
        - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call
        - windowscsehelper.ps1 contains basic util functions. It will be compressed to a zip file and then be converted to base64 encoding
          string and stored in $zippedFiles. Reason: This script is a template and has some limitations.
        - All other scripts will be packaged and published in a single package. It will be downloaded in provisioning VM.
          Reason: CustomData has length limitation 87380.
        - ProvisioningScriptsPackage contains scripts to start kubelet, kubeproxy, etc. The source is https://github.com/Azure/aks-engine/tree/master/staging/provisioning/windows
#>
[CmdletBinding(DefaultParameterSetName="Standard")]
param(
    [parameter(Mandatory=$true)]
    [ValidateNotNullOrEmpty()]
    $AgentKey,

    [parameter(Mandatory=$true)]
    [ValidateNotNullOrEmpty()]
    $AADClientSecret, # base64

    # C:\AzureData\provision.complete
    # MUST keep generating this file when CSE is done and do not change the name
    #  - It is used to avoid running CSE multiple times
    #  - Some customers use this file to check if CSE is done
    [parameter(Mandatory=$true)]
    [ValidateNotNullOrEmpty()]
    $CSEResultFilePath
)


# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world.
# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation

$MasterIP="{{GetKubernetesEndpoint}}"
$KubeDnsServiceIp="{{GetParameter "kubeDNSServiceIP"}}"
$MasterFQDNPrefix="{{GetParameter "masterEndpointDNSNamePrefix"}}"
$Location="{{GetVariable "location"}}"
{{if UserAssignedIDEnabled}}
$UserAssignedClientID="{{GetVariable "userAssignedIdentityID"}}"
{{end}}
$TargetEnvironment="{{GetTargetEnvironment}}"
$ArmResourceEndpoint="{{GetArmResourceEndpoint}}"
$AADClientId="{{GetParameter "servicePrincipalClientId"}}"
$NetworkAPIVersion="2018-08-01"

# Do not parse the start time from $LogFile to simplify the logic
$StartTime=Get-Date
$global:ExitCode=0
$global:ErrorMessage=""

# These globals will not change between nodes in the same cluster, so they are not
# passed as powershell parameters

## SSH public keys to add to authorized_keys
$global:SSHKeys=@( {{GetSshPublicKeysPowerShell}} )

## Certificates generated by aks-engine
$global:CACertificate="{{GetParameter "caCertificate"}}"
$global:AgentCertificate="{{GetParameter "clientCertificate"}}"

## Download sources provided by aks-engine
$global:KubeBinariesPackageSASURL="{{GetParameter "kubeBinariesSASURL"}}"
$global:WindowsKubeBinariesURL="{{GetParameter "windowsKubeBinariesURL"}}"
$global:KubeBinariesVersion="{{GetParameter "kubeBinariesVersion"}}"
$global:ContainerdUrl="{{GetParameter "windowsContainerdURL"}}"
$global:ContainerdSdnPluginUrl="{{GetParameter "windowsSdnPluginURL"}}"

## Docker Version
$global:DockerVersion="{{GetParameter "windowsDockerVersion"}}"

## ContainerD Usage
$global:DefaultContainerdWindowsSandboxIsolation="{{GetParameter "defaultContainerdWindowsSandboxIsolation"}}"
$global:ContainerdWindowsRuntimeHandlers="{{GetParameter "containerdWindowsRuntimeHandlers"}}"

## VM configuration passed by Azure
$global:WindowsTelemetryGUID="{{GetParameter "windowsTelemetryGUID"}}"
{{if eq GetIdentitySystem "adfs"}}
$global:TenantId="adfs"
{{else}}
$global:TenantId="{{GetVariable "tenantID"}}"
{{end}}
$global:SubscriptionId="{{GetVariable "subscriptionId"}}"
$global:ResourceGroup="{{GetVariable "resourceGroup"}}"
$global:VmType="{{GetVariable "vmType"}}"
$global:SubnetName="{{GetVariable "subnetName"}}"
# NOTE: MasterSubnet is still referenced by `kubeletstart.ps1` and `windowsnodereset.ps1`
# for case of Kubenet
$global:MasterSubnet=""
$global:SecurityGroupName="{{GetVariable "nsgName"}}"
$global:VNetName="{{GetVariable "virtualNetworkName"}}"
$global:RouteTableName="{{GetVariable "routeTableName"}}"
$global:PrimaryAvailabilitySetName="{{GetVariable "primaryAvailabilitySetName"}}"
$global:PrimaryScaleSetName="{{GetVariable "primaryScaleSetName"}}"

$global:KubeClusterCIDR="{{GetParameter "kubeClusterCidr"}}"
$global:KubeServiceCIDR="{{GetParameter "kubeServiceCidr"}}"
$global:VNetCIDR="{{GetParameter "vnetCidr"}}"
{{if IsKubernetesVersionGe "1.16.0"}}
$global:KubeletNodeLabels="{{GetAgentKubernetesLabels .}}"
{{else}}
$global:KubeletNodeLabels="{{GetAgentKubernetesLabelsDeprecated .}}"
{{end}}
$global:KubeletConfigArgs=@( {{GetKubeletConfigKeyValsPsh}} )
$global:KubeletHealthzEndpoint="{{GetKubeletHealthzEndpoint}}"
$global:KubeproxyConfigArgs=@( {{GetKubeproxyConfigKeyValsPsh}} )

$global:KubeproxyFeatureGates=@( {{GetKubeProxyFeatureGatesPsh}} )

$global:UseManagedIdentityExtension="{{GetVariable "useManagedIdentityExtension"}}"
$global:UseInstanceMetadata="{{GetVariable "useInstanceMetadata"}}"

$global:LoadBalancerSku="{{GetVariable "loadBalancerSku"}}"
$global:ExcludeMasterFromStandardLB="{{GetVariable "excludeMasterFromStandardLB"}}"

$global:PrivateEgressProxyAddress="{{GetPrivateEgressProxyAddress}}"

# Windows defaults, not changed by aks-engine
$global:CacheDir="c:\akse-cache"
$global:KubeDir="c:\k"
$global:HNSModule=[Io.path]::Combine("$global:KubeDir", "hns.v2.psm1")

$global:KubeDnsSearchPath="svc.cluster.local"

$global:CNIPath=[Io.path]::Combine("$global:KubeDir", "cni")
$global:NetworkMode="L2Bridge"
$global:CNIConfig=[Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf")
$global:CNIConfigPath=[Io.path]::Combine("$global:CNIPath", "config")


$global:AzureCNIDir=[Io.path]::Combine("$global:KubeDir", "azurecni")
$global:AzureCNIBinDir=[Io.path]::Combine("$global:AzureCNIDir", "bin")
$global:AzureCNIConfDir=[Io.path]::Combine("$global:AzureCNIDir", "netconf")

# Azure cni configuration
# $global:NetworkPolicy="{{GetParameter "networkPolicy"}}" # BUG: unused
$global:NetworkPlugin="{{GetParameter "networkPlugin"}}"
$global:VNetCNIPluginsURL="{{GetParameter "vnetCniWindowsPluginsURL"}}"
$global:IsDualStackEnabled={{if IsIPv6DualStackFeatureEnabled}}$true {{else}}$false {{end}}
$global:IsAzureCNIOverlayEnabled={{if IsAzureCNIOverlayFeatureEnabled}}$true {{else}}$false {{end}}
$global:CiliumDataplaneEnabled={{if CiliumDataplaneEnabled}}$true {{else}}$false {{end}}
$global:IsIMDSRestrictionEnabled={{if EnableIMDSRestriction}}$true {{else}}$false {{end}}

# Kubelet credential provider
$global:CredentialProviderURL="{{GetParameter "windowsCredentialProviderURL"}}"

# CSI Proxy settings
$global:EnableCsiProxy=[System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy"}}");
$global:CsiProxyUrl="{{GetVariable "windowsCSIProxyURL"}}";

# Hosts Config Agent settings
$global:EnableHostsConfigAgent=[System.Convert]::ToBoolean("{{EnableHostsConfigAgent}}");

# These scripts are used by cse
$global:CSEScriptsPackageUrl="{{GetVariable "windowsCSEScriptsPackageURL"}}";

# The windows nvidia gpu driver related url is used by windows cse
$global:GpuDriverURL="{{GetVariable "windowsGpuDriverURL"}}";

# PauseImage
$global:WindowsPauseImageURL="{{GetVariable "windowsPauseImageURL"}}";
$global:AlwaysPullWindowsPauseImage=[System.Convert]::ToBoolean("{{GetVariable "alwaysPullWindowsPauseImage"}}");

# Calico
$global:WindowsCalicoPackageURL="{{GetVariable "windowsCalicoPackageURL"}}";

## GPU install
$global:ConfigGPUDriverIfNeeded=[System.Convert]::ToBoolean("{{GetVariable "configGPUDriverIfNeeded"}}");

# GMSA
$global:WindowsGmsaPackageUrl="{{GetVariable "windowsGmsaPackageUrl"}}";

# TLS Bootstrap Token
$global:TLSBootstrapToken="{{GetTLSBootstrapTokenForKubeConfig}}"

# Secure TLS Bootstrap settings
$global:EnableSecureTLSBootstrapping=[System.Convert]::ToBoolean("{{EnableSecureTLSBootstrapping}}");
$global:SecureTLSBootstrappingAADResource="{{GetSecureTLSBootstrappingAADResource}}";
$global:SecureTLSBootstrappingUserAssignedIdentityID="{{GetSecureTLSBootstrappingUserAssignedIdentityID}}";
$global:CustomSecureTLSBootstrappingClientDownloadURL="{{GetCustomSecureTLSBootstrappingClientDownloadURL}}";
$global:SecureTLSBootstrappingValidateKubeconfigTimeout="{{GetSecureTLSBootstrappingValidateKubeconfigTimeout}}";
$global:SecureTLSBootstrappingGetAccessTokenTimeout="{{GetSecureTLSBootstrappingGetAccessTokenTimeout}}";
$global:SecureTLSBootstrappingGetInstanceDataTimeout="{{GetSecureTLSBootstrappingGetInstanceDataTimeout}}";
$global:SecureTLSBootstrappingGetNonceTimeout="{{GetSecureTLSBootstrappingGetNonceTimeout}}";
$global:SecureTLSBootstrappingGetAttestedDataTimeout="{{GetSecureTLSBootstrappingGetAttestedDataTimeout}}";
$global:SecureTLSBootstrappingGetCredentialTimeout="{{GetSecureTLSBootstrappingGetCredentialTimeout}}";

# uniquely identifies AKS's Entra ID application, see: https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication#how-to-use-kubelogin-with-aks
# this is used by aks-secure-tls-bootstrap-client.exe when requesting AAD tokens
# TODO(cameissner): remove once 2025-10B image is released
$global:AKSAADServerAppID="6dae42f8-4368-4678-94ff-3960e28e3630"

# Disable OutBoundNAT in Azure CNI configuration
$global:IsDisableWindowsOutboundNat=[System.Convert]::ToBoolean("{{GetVariable "isDisableWindowsOutboundNat"}}");

# Base64 representation of ZIP archive
$zippedFiles="{{GetKubernetesWindowsAgentFunctions}}"

$global:KubeClusterConfigPath="c:\k\kubeclusterconfig.json"
$fipsEnabled=[System.Convert]::ToBoolean("{{FIPSEnabled}}")

# HNS remediator
$global:HNSRemediatorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetHnsRemediatorIntervalInMinutes}}");

# Log generator
$global:LogGeneratorIntervalInMinutes=[System.Convert]::ToUInt32("{{GetLogGeneratorIntervalInMinutes}}");

$global:EnableIncreaseDynamicPortRange=$false

$global:RebootNeeded=$false

$global:IsSkipCleanupNetwork=[System.Convert]::ToBoolean("{{GetVariable "isSkipCleanupNetwork"}}");
$PreProvisionOnly=[System.Convert]::ToBoolean("{{GetPreProvisionOnly}}");

$global:EnableKubeletServingCertificateRotation=[System.Convert]::ToBoolean("{{EnableKubeletServingCertificateRotation}}")

# Windows Cilium Networking (WCN) Platform configuration
$global:EnableWindowsCiliumNetworking=[System.Convert]::ToBoolean("{{GetVariable "nextGenNetworkingEnabled"}}");
$global:WindowsCiliumNetworkingConfiguration="{{GetVariable "nextGenNetworkingConfig"}}";
$global:WindowsCiliumNetworkingPath=Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath=Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}"
$global:MCRRepositoryBase="{{GetMCRRepositoryBase}}"
$global:NetworkIsolatedClusterTestMode=[System.Convert]::ToBoolean("{{GetNetworkIsolatedClusterTestMode}}"); # for ab e2e only for local ab test with remote cse package

$global:OrasCacheDir="c:\aks-tools\oras\" # refer to components.json
$global:OrasPath="c:\aks-tools\oras\oras.exe"
$global:OrasOutput="c:\aks-tools\oras\oras_verbose.out"
$global:OrasRegistryConfigFile="c:\aks-tools\oras\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined"
$global:OperationId=New-Guid

# Extract cse helper script from ZIP
function Get-HelperScripts {
    [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
    try {
        Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force -ErrorAction Stop
        # Dot-source windowscsehelper.ps1 with functions that are called in this script
        . c:\AzureData\windows\windowscsehelper.ps1
        # util functions only can be used after this line, for example, Write-Log
    }
    catch {
        $global:ErrorMessage=("Failed to extract inline scripts.zip: $($_.Exception.Message)" -replace '\|', '%7C' )
        $global:ExitCode=73
        exit 73
    }

    # Detect whether the operator has supplied a fully-qualified CSE scripts zip URL
    # (as opposed to a base URL ending in "/"). A fully-qualified URL is a signal to
    # *always* fetch and overwrite the cached scripts on the VHD, which is required
    # for testing branch builds against baked VHDs that already contain a prior
    # version of the CSE scripts. Production RP only ever sets a base URL ending in
    # "/", so this branch is a no-op for production traffic.
    $global:IsExplicitCSEScriptsPackageUrl=$false
    if (-not [string]::IsNullOrWhiteSpace($global:CSEScriptsPackageUrl) -and -not $global:CSEScriptsPackageUrl.EndsWith("/")) {
        $global:IsExplicitCSEScriptsPackageUrl=$true
    }

    if ((-not (Test-Path "C:\AzureData\windows\azurecnifunc.ps1")) -or $global:IsExplicitCSEScriptsPackageUrl) {
        # CSEScriptsPackage is cached on VHD. Previously the cse package version was managed in components.json, whereas RP set the package URL which is a storage account.
        # From 2025-06 The CSE packages is released on the VHD. RP can use fully qualified URL to download CSE scripts package when required out of VHD release cycle.
        # In the transition period, it is important that when deal with older VHD versions, the agentbaker runtime provision script needs to be compatible with the latest known storage account package, 0.0.52.

        $WindowsCSEScriptsPackage="aks-windows-cse-scripts-current.zip"
        $scriptsZip=$null
        $shouldCleanup=$false

        # Step 1: Try to find cached scripts on VHD (skipped when an explicit
        # CSEScriptsPackageUrl was supplied, so the operator-provided zip is the
        # source of truth and always overwrites the cached copy).
        if (-not $global:IsExplicitCSEScriptsPackageUrl -and $global:CacheDir -and (Test-Path $global:CacheDir)) {
            $searchCachedScripts=[IO.Directory]::GetFiles($global:CacheDir, $WindowsCSEScriptsPackage, [IO.SearchOption]::AllDirectories)
            Write-Log "the directory $global:CacheDir contains the following files:"
            Get-ChildItem -Path $global:CacheDir | ForEach-Object { Write-Log "  $_" }
            if ($searchCachedScripts.Count -gt 0) {
                $scriptsZip=$searchCachedScripts[0]
                Write-Log "Found cached CSE scripts at $scriptsZip"
            }
        }

        # Step 2: For non-network-isolated clusters, download scripts if needed (overrides cached version when appropriate)
        $isNetworkIsolated=-not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer) -and -not $global:NetworkIsolatedClusterTestMode
        if (-not $isNetworkIsolated) {
            Write-Log "Requested CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl"
            if ($global:CSEScriptsPackageUrl.EndsWith("/")) {
                if (-not $scriptsZip) {
                    Write-Log "Could not find windows cse package on VHD. Use remote version instead."
                    $WindowsCSEScriptsPackage="aks-windows-cse-scripts-v0.0.52.zip"
                }
                Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage"
                $global:CSEScriptsPackageUrl=$global:CSEScriptsPackageUrl + $WindowsCSEScriptsPackage
            }
            Write-Log "CSEScriptsPackageUrl used for provision is $global:CSEScriptsPackageUrl"

            # Download CSE function scripts
            $downloadedFile='c:\csescripts.zip'
            Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl"
            DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $downloadedFile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE
            $scriptsZip=$downloadedFile
            $shouldCleanup=$true
        }
        else {
            Write-Log "Network isolated cluster detected (BootstrapProfileContainerRegistryServer is set), skip CSE scripts download and use cached scripts"
            if (-not $scriptsZip) {
                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_NETWORK_ISOLATED_CLUSTER_CSE_NOT_CACHED -ErrorMessage "Cached CSE scripts package '$WindowsCSEScriptsPackage' not found under cache directory '$global:CacheDir'"
            }
        }

        # Step 3: Extract scripts from the resolved zip
        Write-Log "Extracting CSE scripts from $scriptsZip"
        AKS-Expand-Archive -Path $scriptsZip -DestinationPath "C:\\AzureData\\windows"
        if ($shouldCleanup) {
            Remove-Item -Path $scriptsZip -Force
        }
    }
    else {
        Write-Log "CSE scripts already exist and no explicit CSEScriptsPackageUrl override, skipping download"
    }

    if (Test-Path -Path 'c:\AzureData\windows\all.ps1') {
        . c:\AzureData\windows\all.ps1
        return
    }

    Write-Log "All CSE function script not found, falling back to individual import"

    # Dot-source cse scripts with functions that are called in this script

    # Dot-source cse scripts with functions that are called in this script
    . c:\AzureData\windows\azurecnifunc.ps1
    . c:\AzureData\windows\calicofunc.ps1
    . c:\AzureData\windows\configfunc.ps1
    . c:\AzureData\windows\containerdfunc.ps1
    . c:\AzureData\windows\kubeletfunc.ps1
    . c:\AzureData\windows\kubernetesfunc.ps1
    . c:\AzureData\windows\nvidiagpudriverfunc.ps1

    $optionalFunctionScripts = @(
        'c:\AzureData\windows\securetlsbootstrapfunc.ps1'
        'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1'
        'c:\AzureData\windows\networkisolatedclusterfunc.ps1'
    )

    foreach ($script in $optionalFunctionScripts) {
        if (Test-Path -Path $script) {
            . $script
        } else {
            Write-Log "$script function script not found, skipping dot-source"
        }
    }
}

# ====== BASE PREP: BASE IMAGE PREPARATION ======
# All operations that prepare the base VHD image
function BasePrep {
    Write-Log "Starting BasePrep - Base image preparation"
    Logs-To-Event -TaskName "AKS.WindowsCSE.BasePrep" -TaskMessage "Starting BasePrep - Base image preparation"

    # This involves using proxy, log the config before fetching packages
    Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'"
    # TODO update to use proxy

    # Install OpenSSH if SSH enabled
    $sshEnabled=[System.Convert]::ToBoolean("{{WindowsSSHEnabled}}")
    if ( $sshEnabled ) {
        Install-OpenSSH -SSHKeys $SSHKeys
    }

    Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID

    Resize-OSDrive

    Initialize-DataDisks

    Initialize-DataDirectories

    Logs-To-Event -TaskName "AKS.WindowsCSE.GetProvisioningAndLogCollectionScripts" -TaskMessage "Start to get provisioning scripts and log collection scripts"
    Create-Directory -FullPath "c:\k"
    Write-Log "Remove `"NT AUTHORITY\Authenticated Users`" write permissions on files in c:\k"
    icacls.exe "c:\k" /inheritance:r
    if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to set inheritance on c:\k (exit code $LASTEXITCODE)" }
    icacls.exe "c:\k" /grant:r SYSTEM:`(OI`)`(CI`)`(F`)
    if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant SYSTEM permissions on c:\k (exit code $LASTEXITCODE)" }
    icacls.exe "c:\k" /grant:r BUILTIN\Administrators:`(OI`)`(CI`)`(F`)
    if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Administrators permissions on c:\k (exit code $LASTEXITCODE)" }
    icacls.exe "c:\k" /grant:r BUILTIN\Users:`(OI`)`(CI`)`(RX`)
    if ($LASTEXITCODE -ne 0) { throw "icacls.exe failed to grant Users permissions on c:\k (exit code $LASTEXITCODE)" }
    Write-Log "c:\k permissions: "
    icacls.exe "c:\k"
    Get-ProvisioningScripts
    Get-LogCollectionScripts

    # kubeclusterconfig.json must exist before Install-Containerd-Based-On-Kubernetes-Version
    # runs below: the CSE scripts package cached on the VHD (e.g. v0.0.52) reads the pause image
    # from this file while configuring containerd during BasePrep. The cluster-specific values
    # written here are only placeholders for the bake/non-cached containerd read - NodePrep
    # rewrites this file from live CustomData on every provision (including PIS/VHD-cached nodes
    # where BasePrep is skipped), so the runtime always sees fresh values.
    Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp

    # oras initialization, including install and login, must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version
    if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and
        -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) {
        # variable exists and is not empty/whitespace
        if (Get-Command -Name Initialize-Oras -ErrorAction SilentlyContinue) {
            Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeOras" -TaskMessage "Ensure oras is initialized for network isolated cluster"
            Initialize-Oras
        }
        else {
            Write-Log "Initialize-Oras is not a recognized function, will skip oras initialization for network isolated cluster"
        }
    }

    # to ensure we don't introduce any incompatibility between base CSE + CSE package versions
    if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) {
        Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL
    }
    else {
        Write-Log "Install-SecureTLSBootstrapClient is not a recognized function, will skip installation of the secure TLS bootstrap client"
    }

    Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{AKSCustomCloudContainerRegistryDNSSuffix}}" {{else}}"" {{end}}

    Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

    $cniBinPath=$global:AzureCNIBinDir
    $cniConfigPath=$global:AzureCNIConfDir
    if ($global:NetworkPlugin -eq "kubenet") {
        $cniBinPath=$global:CNIPath
        $cniConfigPath=$global:CNIConfigPath
    }

    Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir -KubernetesVersion $global:KubeBinariesVersion

    Retag-ImagesForAzureChinaCloud -TargetEnvironment $TargetEnvironment

    if ($global:EnableCsiProxy) {
        New-CsiProxyService -CsiProxyPackageUrl $global:CsiProxyUrl -KubeDir $global:KubeDir
    }

    if ($global:EnableHostsConfigAgent) {
        New-HostsConfigService
    }

    Set-Explorer
    Adjust-PageFileSize
    Logs-To-Event -TaskName "AKS.WindowsCSE.PreprovisionExtension" -TaskMessage "Start preProvisioning script"
    PREPROVISION_EXTENSION
    Adjust-DynamicPortRange
    Register-LogsCleanupScriptTask

    Update-DefenderPreferences

    $windowsVersion=Get-WindowsVersion
    if ($windowsVersion -eq "1809") {
        Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Start to enable secure TLS protocols"
        try {
            . C:\k\windowssecuretls.ps1
            Enable-SecureTls
        }
        catch {
            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS -ErrorMessage $_
        }
    } else {
        Logs-To-Event -TaskName "AKS.WindowsCSE.EnableSecureTLS" -TaskMessage "Skip secure TLS protocols for Windows version: $windowsVersion"
    }

    Enable-FIPSMode -FipsEnabled $fipsEnabled
    if ($global:WindowsGmsaPackageUrl) {
        Install-GmsaPlugin -GmsaPackageUrl $global:WindowsGmsaPackageUrl
    }

    Write-Log "BasePrep completed successfully"
    Logs-To-Event -TaskName "AKS.WindowsCSE.BasePrep" -TaskMessage "BasePrep completed successfully"
}

# ====== NODE PREP: CLUSTER INTEGRATION ======
# All operations that should only run when connecting to the actual cluster
function NodePrep {
    # Write the kubeconfigs here, not in BasePrep: BasePrep is skipped on
    # PIS/VHD-cached nodes, so anything written there is baked with stale
    # bake-time values. Both are only consumed in NodePrep, so write them from
    # live values on every provision before their consumers run below.
    if ($global:TLSBootstrapToken) {
        Write-BootstrapKubeConfig -CACertificate $global:CACertificate `
            -KubeDir $global:KubeDir `
            -MasterFQDNPrefix $MasterFQDNPrefix `
            -MasterIP $MasterIP `
            -TLSBootstrapToken $global:TLSBootstrapToken
    }

    if ($global:TLSBootstrapToken -or $global:EnableSecureTLSBootstrapping) {
        # NOTE: we need kubeconfig to setup calico even if vanilla/secure TLS bootstrapping is enabled
        # This kubeconfig will be deleted after calico installation.
        Write-Log "Write temporary kube config"
    } else {
        Write-Log "Write kube config"
    }

    Write-KubeConfig -CACertificate $global:CACertificate `
        -KubeDir $global:KubeDir `
        -MasterFQDNPrefix $MasterFQDNPrefix `
        -MasterIP $MasterIP `
        -AgentKey $AgentKey `
        -AgentCertificate $global:AgentCertificate

    # Write cluster-specific configuration files from live CustomData.
    # PIS images are reusable across clusters, so these files must not be baked
    # into the VHD during BasePrep.
    #
    # NOTE: Configure-KubeletServingCertificateRotation MUST run before
    # Write-KubeClusterConfig (it mutates kubelet config args and node labels).
    # Write-KubeClusterConfig also runs in BasePrep (the cached CSE scripts read
    # the pause image from it while configuring containerd); we rewrite it here so
    # cluster-specific values are refreshed on every provision, including
    # PIS/VHD-cached nodes where BasePrep is skipped.
    Configure-KubeletServingCertificateRotation
    Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp

    Write-AzureConfig `
        -KubeDir $global:KubeDir `
        -AADClientId $AADClientId `
        -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `
        -TenantId $global:TenantId `
        -SubscriptionId $global:SubscriptionId `
        -ResourceGroup $global:ResourceGroup `
        -Location $Location `
        -VmType $global:VmType `
        -SubnetName $global:SubnetName `
        -SecurityGroupName $global:SecurityGroupName `
        -VNetName $global:VNetName `
        -RouteTableName $global:RouteTableName `
        -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName `
        -PrimaryScaleSetName $global:PrimaryScaleSetName `
        -UseManagedIdentityExtension $global:UseManagedIdentityExtension `
        -UserAssignedClientID $UserAssignedClientID `
        -UseInstanceMetadata $global:UseInstanceMetadata `
        -LoadBalancerSku $global:LoadBalancerSku `
        -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB `
        -TargetEnvironment {{if IsAKSCustomCloud}}"AzureStackCloud" {{else}}$TargetEnvironment {{end}}

    # we borrow the logic of AzureStackCloud to achieve AKSCustomCloud.
    # In case of AKSCustomCloud, customer cloud env will be loaded from azurestackcloud.json
    {{if IsAKSCustomCloud}}
    $azureStackConfigFile=[io.path]::Combine($global:KubeDir, "azurestackcloud.json")
    $envJSON="{{GetBase64EncodedEnvironmentJSON}}"
    [io.file]::WriteAllBytes($azureStackConfigFile, [System.Convert]::FromBase64String($envJSON))
    {{end}}

    # Feature-detect Get-CACertificates params for VHD/CSE-zip skew.
    # Older CSE packages (pre-rcv1p) have Get-CACertificates with no parameters;
    # newer versions accept -Location (optional) and -FailOnError. Bind only the
    # parameters the resolved function actually supports so a new VHD paired with
    # an older CSE zip does not fail with parameter-binding errors.
    $getCACertsCmd=Get-Command -Name Get-CACertificates -ErrorAction Ignore
    $getCACertsArgs=@{}
    if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('Location')) {
        $getCACertsArgs['Location']=$Location
    }
    if ($getCACertsCmd -and $getCACertsCmd.Parameters.ContainsKey('FailOnError')) {
        $getCACertsArgs['FailOnError']=$true
    }
    Get-CACertificates @getCACertsArgs

    Write-CACert -CACertificate $global:CACertificate `
        -KubeDir $global:KubeDir

    Install-KubernetesServices -KubeDir $global:KubeDir
    Update-ServiceFailureActions

    Write-Log "Starting NodePrep - Cluster integration"
    Logs-To-Event -TaskName "AKS.WindowsCSE.NodePrep" -TaskMessage "Starting NodePrep - Cluster integration"

    Check-APIServerConnectivity -MasterIP $MasterIP

    Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin"

    # Configure network policy.
    Get-HnsPsm1 -HNSModule $global:HNSModule
    Import-Module $global:HNSModule

    Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir `
        -AzureCNIBinDir $global:AzureCNIBinDir `
        -VNetCNIPluginsURL $global:VNetCNIPluginsURL

    Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir `
        -KubeDnsSearchPath $global:KubeDnsSearchPath `
        -KubeClusterCIDR $global:KubeClusterCIDR `
        -KubeServiceCIDR $global:KubeServiceCIDR `
        -VNetCIDR $global:VNetCIDR `
        -IsDualStackEnabled $global:IsDualStackEnabled `
        -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled

    if ($TargetEnvironment -ieq "AzureStackCloud") {
        GenerateAzureStackCNIConfig `
            -TenantId $global:TenantId `
            -SubscriptionId $global:SubscriptionId `
            -ResourceGroup $global:ResourceGroup `
            -AADClientId $AADClientId `
            -KubeDir $global:KubeDir `
            -AADClientSecret $([System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($AADClientSecret))) `
            -NetworkAPIVersion $NetworkAPIVersion `
            -AzureEnvironmentFilePath $([io.path]::Combine($global:KubeDir, "azurestackcloud.json")) `
            -IdentitySystem "{{GetIdentitySystem}}"
    }

    New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled

    # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this)
    netsh advfirewall set allprofiles state off
    if ($LASTEXITCODE -ne 0) { throw "netsh advfirewall failed to disable firewall (exit code $LASTEXITCODE)" }

    # To ensure we don't introduce any incompatibility between base CSE + CSE package versions
    if (Get-Command -Name Enable-WindowsCiliumNetworking -ErrorAction SilentlyContinue) {
        if ($global:EnableWindowsCiliumNetworking) {
            Enable-WindowsCiliumNetworking
        }
        else {
            Write-Log "Windows Cilium Networking is not enabled, will skip Windows Cilium Networking installation"
        }
    }
    else {
        Write-Log "Enable-WindowsCiliumNetworking is not a recognized function, will skip Windows Cilium Networking installation"
    }

    if ($global:WindowsCalicoPackageURL) {
        Start-InstallCalico -RootDir "c:\" -KubeServiceCIDR $global:KubeServiceCIDR -KubeDnsServiceIp $KubeDnsServiceIp
    }
    if ($global:TLSBootstrapToken -or $global:EnableSecureTLSBootstrapping) {
        Write-Log "Removing temporary kube config"
        $kubeConfigFile=[io.path]::Combine($KubeDir, "config")
        Remove-Item $kubeConfigFile
    }

    # Register AFTER temp kubeconfig removal: the -AtStartup trigger would
    # otherwise race PIS-baked VHD first boot and bring kubelet up with the
    # embedded "nodeclient" cert instead of doing TLS bootstrap.
    Register-NodeResetScriptTask

    # Guard against older CSE packages that do not yet export Should-InstallCACertificatesRefreshTask.
    # If the function is absent (old package), fall back to the previous unconditional behaviour so
    # that legacy/ussec/usnat clusters continue to register the refresh task.
    if (Get-Command -Name Should-InstallCACertificatesRefreshTask -ErrorAction Ignore) {
        if (Should-InstallCACertificatesRefreshTask -Location $Location) {
            Register-CACertificatesRefreshTask -Location $Location
        }
    } elseif (Get-Command -Name Register-CACertificatesRefreshTask -ErrorAction Ignore) {
        Register-CACertificatesRefreshTask -Location $Location
    }

    Start-InstallGPUDriver -EnableInstall $global:ConfigGPUDriverIfNeeded -GpuDriverURL $global:GpuDriverURL

    if (Test-Path $CacheDir) {
        Write-Log "Removing aks cache directory"
        Remove-Item $CacheDir -Recurse -Force
    }

    Enable-GuestVMLogs -IntervalInMinutes $global:LogGeneratorIntervalInMinutes

    if ($global:RebootNeeded) {
        Logs-To-Event -TaskName "AKS.WindowsCSE.RestartComputer" -TaskMessage "Setup Complete, calling Postpone-RestartComputer with reboot"
        Postpone-RestartComputer
    }
    else {
        Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot"
        Start-NodeResetScriptTask
    }

    Write-Log "NodePrep completed successfully"
    Logs-To-Event -TaskName "AKS.WindowsCSE.NodePrep" -TaskMessage "NodePrep completed successfully"
}

try {
    . Get-HelperScripts

    Logs-To-Event -TaskName "AKS.WindowsCSE.ExecuteCustomDataSetupScript" -TaskMessage ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AADClientId $AADClientId -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment -CSEResultFilePath $CSEResultFilePath"

    # Exit early if the script has been executed
    if (Test-Path -Path $CSEResultFilePath -PathType Leaf) {
        Write-Log "The script has been executed before, will exit without doing anything."
        return
    }

    # The provisioning is split into two stages to support VHD image creation workflows:
    #
    # Stage 1: Base image preparation
    #   - Installs and configures all required components (kubelet, containerd, etc.)
    #   - Sets up system configurations that are common across all nodes
    #   - DOES NOT join the node to any cluster
    #   - After this stage, users can add customizations (e.g., pre-pull additional container images)
    #   - The VM can then be captured as a VHD image for use as a node pool base image
    #
    # Stage 2: Cluster integration and hardware setup
    #   - Performs cluster-specific configurations
    #   - Configures hardware-specific components (GPU drivers, MIG partitions, etc.)
    #   - Establishes connection to the API server
    #   - Joins the node to the cluster
    #   - Only runs when actually provisioning a node, not when creating VHD images
    #
    # In typical deployments, both stages run sequentially during node provisioning.
    # For VHD image creation workflows, only basePrep runs initially, and nodePrep runs later
    # when nodes are created from that VHD image.
    if (-not (Test-Path "C:\AzureData\base_prep.complete")) {
        BasePrep
    }
    else {
        Write-Log "Skipping basePrep - base_prep.complete file exists"
    }
    if (-not $PreProvisionOnly) {
        NodePrep
    }
    else {
        Write-Log "Skipping nodePrep - pre-provision only mode"
    }
}
catch {
    Resolve-Error
    # Set-ExitCode will exit with the specified ExitCode immediately and not be caught by this catch block
    # Ideally all exceptions will be handled and no exception will be thrown.
    Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_UNKNOWN -ErrorMessage $_
}
finally {
    # Generate CSE result so it can be returned as the CSE response in csecmd.ps1
    $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date))
    Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode"

    Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage."

    # Create appropriate completion file based on mode
    $completionFilePath=if ($PreProvisionOnly) { "C:\AzureData\base_prep.complete" } else { $CSEResultFilePath }

    if ($global:ExitCode -eq 0) {
        Set-Content -Path $completionFilePath -Value $global:ExitCode -Force
    }
    else {
        # $JsonString="ExitCode: |{0}|, Output: |{1}|, Error: |{2}|"
        # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe.
        $errorMessageLength="ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: ||".Length
        $turncatedErrorMessage=$global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length))
        Set-Content -Path $completionFilePath -Value "ExitCode: |$global:ExitCode|, Output: |$($global:ErrorCodeNames[$global:ExitCode])|, Error: |$turncatedErrorMessage|"
    }

    # Upload-GuestVMLogs is defined in the CSE scripts package (configfunc.ps1).
    # If the CSE scripts package failed to download or load, the function will not be available.
    if (Get-Command -Name Upload-GuestVMLogs -ErrorAction SilentlyContinue) {
        Upload-GuestVMLogs -ExitCode $global:ExitCode
    }
    else {
        Write-Log "Upload-GuestVMLogs is not available, skipping log upload (CSE scripts package may not have been loaded)"
    }
}