Document Details

# set variables set AZURE_STORAGE_ACCOUNT=STORAGEACCOUNT set AZURE_STORAGE_CONTAINER=STORAGECONTAINER #Login az login # If you have multiple subscriptions, set the one to use # az account set --subscription SUBSCRIPTION # Retrieve the primary key for the storage account az storage account keys list --account-name %AZURE_STORAGE_ACCOUNT% --query "[0].{PrimaryKey:value}" --output table

#set variable for primary key set AZURE_STORAGE_KEY=PRIMARYKEY

$resourceGroupName = "RESOURCEGROUP" $storageAccountName = "STORAGEACCOUNT" $containerName = "STORAGECONTAINER" $policy = "myPolicyPS" # Login to your Azure subscription $sub = Get-AzSubscription -ErrorAction SilentlyContinue if(-not($sub)) { Connect-AzAccount } # If you have multiple subscriptions, set the one to use # Select-AzSubscription -SubscriptionId "<SUBSCRIPTIONID>" # Get the access key for the Azure Storage account $storageAccountKey = (Get-AzStorageAccountKey ` -ResourceGroupName $resourceGroupName ` -Name $storageAccountName)[0].Value # Create an Azure Storage context $storageContext = New-AzStorageContext ` -StorageAccountName $storageAccountName ` -StorageAccountKey $storageAccountKey # Create a stored access policy for the Azure storage container New-AzStorageContainerStoredAccessPolicy ` -Container $containerName ` -Policy $policy ` -Permission "rl" ` -ExpiryTime "12/31/2025 08:00:00" ` -Context $storageContext # Get the stored access policy or policies for the Azure storage container Get-AzStorageContainerStoredAccessPolicy ` -Container $containerName ` -Context $storageContext # Generates an SAS token for the Azure storage container New-AzStorageContainerSASToken ` -Name $containerName ` -Policy $policy ` -Context $storageContext <# Removes a stored access policy from the Azure storage container Remove-AzStorageContainerStoredAccessPolicy ` -Container $containerName ` -Policy $policy ` -Context $storageContext #> # upload a file for a later example Set-AzStorageblobcontent ` -File "./sampledata/sample.log" ` -Container $containerName ` -Blob "samplePS.log" ` -Context $storageContext

# Create stored access policy on the containing object az storage container policy create --container-name %AZURE_STORAGE_CONTAINER% --name myPolicyCLI --account-key %AZURE_STORAGE_KEY% --account-name %AZURE_STORAGE_ACCOUNT% --expiry 2025-12-31 --permissions rl # List stored access policies on a containing object az storage container policy list --container-name %AZURE_STORAGE_CONTAINER% --account-key %AZURE_STORAGE_KEY% --account-name %AZURE_STORAGE_ACCOUNT% # Generate a shared access signature for the container az storage container generate-sas --name %AZURE_STORAGE_CONTAINER% --policy-name myPolicyCLI --account-key %AZURE_STORAGE_KEY% --account-name %AZURE_STORAGE_ACCOUNT% # Reversal # az storage container policy delete --container-name %AZURE_STORAGE_CONTAINER% --name myPolicyCLI --account-key %AZURE_STORAGE_KEY% --account-name %AZURE_STORAGE_ACCOUNT% # upload a file for a later example az storage blob upload --container-name %AZURE_STORAGE_CONTAINER% --account-key %AZURE_STORAGE_KEY% --account-name %AZURE_STORAGE_ACCOUNT% --name sampleCLI.log --file "./sampledata/sample.log"

$clusterName = 'CLUSTERNAME' $resourceGroupName = 'RESOURCEGROUP' # Replace with the Azure data center you want to the cluster to live in $location = 'eastus' # Replace with the name of the default storage account TO BE CREATED $defaultStorageAccountName = 'DEFAULTSTORAGEACCOUNT' # Replace with the name of the SAS container CREATED EARLIER $SASContainerName = 'STORAGECONTAINER' # Replace with the name of the SAS storage account CREATED EARLIER $SASStorageAccountName = 'STORAGEACCOUNT' # Replace with the SAS token generated earlier $SASToken = 'TOKEN' # Default cluster size (# of worker nodes), version, and type $clusterSizeInNodes = "4" $clusterVersion = "3.6" $clusterType = "Hadoop" # Login to your Azure subscription $sub = Get-AzSubscription -ErrorAction SilentlyContinue if(-not($sub)) { Connect-AzAccount } # If you have multiple subscriptions, set the one to use # Select-AzSubscription -SubscriptionId "<SUBSCRIPTIONID>" # Create an Azure Storage account and container New-AzStorageAccount ` -ResourceGroupName $resourceGroupName ` -Name $defaultStorageAccountName ` -Location $location ` -SkuName Standard_LRS ` -Kind StorageV2 ` -EnableHttpsTrafficOnly 1 $defaultStorageAccountKey = (Get-AzStorageAccountKey ` -ResourceGroupName $resourceGroupName ` -Name $defaultStorageAccountName)[0].Value $defaultStorageContext = New-AzStorageContext ` -StorageAccountName $defaultStorageAccountName ` -StorageAccountKey $defaultStorageAccountKey # Create a blob container. This holds the default data store for the cluster. New-AzStorageContainer ` -Name $clusterName ` -Context $defaultStorageContext # Cluster login is used to secure HTTPS services hosted on the cluster $httpCredential = Get-Credential ` -Message "Enter Cluster login credentials" ` -UserName "admin" # SSH user is used to remotely connect to the cluster using SSH clients $sshCredential = Get-Credential ` -Message "Enter SSH user credentials" ` -UserName "sshuser" # Create the configuration for the cluster $config = New-AzHDInsightClusterConfig $config = $config | Add-AzHDInsightConfigValue ` -Spark2Defaults @{} ` -Core @{"fs.azure.sas.$SASContainerName.$SASStorageAccountName.blob.core.windows.net"=$SASToken} # Create the HDInsight cluster New-AzHDInsightCluster ` -Config $config ` -ResourceGroupName $resourceGroupName ` -ClusterName $clusterName ` -Location $location ` -ClusterSizeInNodes $clusterSizeInNodes ` -ClusterType $clusterType ` -OSType Linux ` -Version $clusterVersion ` -HttpCredential $httpCredential ` -SshCredential $sshCredential ` -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` -DefaultStorageAccountKey $defaultStorageAccountKey ` -DefaultStorageContainer $clusterName <# REVERSAL Remove-AzHDInsightCluster ` -ResourceGroupName $resourceGroupName ` -ClusterName $clusterName Remove-AzStorageContainer ` -Name $clusterName ` -Context $defaultStorageContext Remove-AzStorageAccount ` -ResourceGroupName $resourceGroupName ` -Name $defaultStorageAccountName Remove-AzResourceGroup ` -Name $resourceGroupName #>