Windows Server 2012 R2 Storage: SMB Scale-Out and Shared VHDX
Here are the details about the names, roles and IP addresses for each of the computers involved:
VM
Computer Role
Net2 (DC/DNS)
Host HOST2
Hyper-V Host
192.168.100.1/24
VM1 CBC-D
DNS, DC
192.168.100.100/24
VM2 CBC-A1
Cluster Node 1
192.168.100.101/24
VM3 CBC-A2
Cluster Node 2
192.168.100.102/24
VM4 CBC-A3
Cluster Node 3
192.168.100.103/24
VM5 CBC-V
VMM / Client
192.168.100.104/24
CBC-A
Cluster
192.168.100.110/24
CBC-F
Scale-Out File Server N/A
1) Configure the physical host
# Preparation steps: Install WS2012R2, rename computer, enable remote desktop
# Prepare base image: Install WS2012R2
C:\windows\system32\sysprep\sysprep.exe /oobe /generalize /shutdown /quiet
# Install required roles and features, restart at the end
Install-WindowsFeature Failover-Clustering -IncludeManagementTools
Install-WindowsFeature Hyper-V -IncludeManagementTools -Restart
# Create a new Internal Switch
New-VMSwitch -Name Net2 -SwitchType Internal
Get-NetAdapter *Net2* | Rename-NetAdapter -NewName vNet2
# Configure vNet2 (parent port for internal switch) with a static IP address for DNS / DC
Set-NetIPInterface -InterfaceAlias vNet2 -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias vNet2 -Confirm:$false
New-NetIPAddress -InterfaceAlias vNet2 -IPAddress 192.168.100.1 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias vNet2 -ServerAddresses 192.168.100.1
# Create 5 VMs, assumes there is a base VM ready at E:\VMS\BaseOS.VHDX
1..5 | % { New-VHD -Path E:\VMS\VM$_.VHDX –ParentPath E:\VMS\BaseOS.VHDX }
1..5 | % { New-VM -Name VM$_ -Path E:\VMS –VHDPath E:\VMS\VM$_.VHDX -Memory 2GB -SwitchName
Net2 }
# Give the VMM VM a little more RAM
Set-VM -VMName VM5 -MemoryStartupBytes 4GB
# Create 3 data VHDX files
1..3 | % { New-VHD -Path E:\VMS\Data$_.VHDX -Fixed -SizeBytes 20GB }
# Manually attach volume E:\
fltMC.exe attach svhdxflt E:\
# NOTE 1: Non-CSV deployments of Shared VHDX are not supported. See the disclaimer section of this
document.
# NOTE 2: Manual attached is not saved across reboots. You will have to re-issue the command after each
reboot.
# Add the 3 data VHDX files to each of the 3 VMs, with Sharing option
1..3 | % { $p = ”E:\VMS\Data” + $_ + ”.VHDX” ; 2..4 | % { $v = ”VM” + $_; Write-Host $v, $p; AddVMHardDiskDrive -VMName $v -Path $p -ShareVirtualDisk } }
2) Configure CBC-D (DNS, DC)
Start-VM –name VM1
# Preparation steps: Install WS2012R2, rename computer
Rename-Computer -NewName CBC-D -Restart
# Install required roles and features, restarts at the end
Install-WindowsFeature AD-Domain-Services, RSAT-ADDS, RSAT-ADDS-Tools
# Configure Internal NIC with a static IP address for DNS / DC
Get-NetAdapter | Rename-NetAdapter -NewName Internal
Set-NetIPInterface -InterfaceAlias Internal -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias Internal -Confirm:$false
New-NetIPAddress -InterfaceAlias Internal -IPAddress 192.168.100.100 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias Internal -ServerAddresses 192.168.100.100
# create AD forest, reboots at the end
Install-ADDSForest -CreateDNSDelegation:$false -DatabasePath "C:\Windows\NTDS" -DomainMode "Win2012"
-DomainName "CBC.TEST" -DomainNetBIOSName "CBC" -ForestMode "Win2012" -InstallDNS:$true -LogPath
"C:\Windows\NTDS" -SYSVOLPath "C:\Windows\SYSVOL"
3) Configure CBC-A1 (Cluster A)
Start-VM –name VM2
# Preparation steps: Install WS2012R2, rename computer
Rename-Computer -NewName CBC-A1 -Restart
# Install required roles and features, restarts at the end
Install-WindowsFeature File-Services, FS-FileServer, Failover-Clustering
Install-WindowsFeature RSAT-Clustering -IncludeAllSubFeature
# Configure Internal NIC with a static IP address for DNS / DC
Get-NetAdapter | Rename-NetAdapter -NewName Internal
Set-NetIPInterface -InterfaceAlias Internal -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias Internal -Confirm:$false
New-NetIPAddress -InterfaceAlias Internal -IPAddress 192.168.100.101 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias Internal -ServerAddresses 192.168.100.100
# Create 1 Pool, 7 Spaces and Initialize them
# NOTE 3: Running Storage Spaces in a guest is not supported. See the disclaimer section of this document.
# NOTE 4: This unsupported configuration cannot simulate Tiering or Write-back cache, since there are no
SSDs.
$s = Get-StorageSubSystem -FriendlyName *Spaces*
New-StoragePool -FriendlyName Pool1 -StorageSubSystemFriendlyName $s.FriendlyName -PhysicalDisks (GetPhysicalDisk -CanPool $true)
Set-ResiliencySetting -Name Mirror -NumberofColumnsDefault 1 -StoragePool (Get-StoragePool FriendlyName Pool1)
New-VirtualDisk -FriendlyName Space1 -StoragePoolFriendlyName Pool1 -ResiliencySettingName Mirror –Size
1GB
2..7 | % { New-VirtualDisk -FriendlyName Space$_ -StoragePoolFriendlyName Pool1 -ResiliencySettingName
Mirror –Size 3GB }
1..7 | % {
$Letter ="PQRSTUV”[($_-1)]
$Number = (Get-VirtualDisk -FriendlyName Space$_ | Get-Disk).Number
Set-Disk -Number $Number -IsReadOnly 0
Set-Disk -Number $Number -IsOffline 0
Initialize-Disk -Number $Number -PartitionStyle MBR
New-Partition -DiskNumber $Number -DriveLetter $Letter -UseMaximumSize
Initialize-Volume -DriveLetter $Letter -FileSystem NTFS -Confirm:$false
}
# Join domain, restart the machine
Add-Computer -DomainName CBC.TEST -Credential (Get-Credential) –Restart
4) Configure CBC-A2 (Cluster A)
Start-VM –name VM3
# Preparation steps: Install WS2012R2, rename computer
Rename-Computer -NewName CBC-A2 -Restart
# Install required roles and features, restarts at the end
Install-WindowsFeature File-Services, FS-FileServer, Failover-Clustering
Install-WindowsFeature RSAT-Clustering -IncludeAllSubFeature
# Configure Internal NIC with a static IP address for DNS / DC
Get-NetAdapter | Rename-NetAdapter -NewName Internal
Set-NetIPInterface -InterfaceAlias Internal -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias Internal -Confirm:$false
New-NetIPAddress -InterfaceAlias Internal -IPAddress 192.168.100.102 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias Internal -ServerAddresses 192.168.100.100
# Join domain, restart the machine
Add-Computer -DomainName CBC.TEST -Credential (Get-Credential) –Restart
5) Configure CBC-A3 (Cluster A)
Start-VM –name VM4
# Preparation steps: Install WS2012R2, rename computer
Rename-Computer -NewName CBC-A3 -Restart
# Install required roles and features, restarts at the end
Install-WindowsFeature File-Services, FS-FileServer, Failover-Clustering
Install-WindowsFeature RSAT-Clustering -IncludeAllSubFeature
# Configure Internal NIC with a static IP address for DNS / DC
Get-NetAdapter | Rename-NetAdapter -NewName Internal
Set-NetIPInterface -InterfaceAlias Internal -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias Internal -Confirm:$false
New-NetIPAddress -InterfaceAlias Internal -IPAddress 192.168.100.103 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias Internal -ServerAddresses 192.168.100.100
# Join domain, restart the machine
Add-Computer -DomainName CBC.TEST -Credential (Get-Credential) -Restart
6) Configure Cluster CBC-A
# Validate cluster
Test-Cluster -Node CBC-A1, CBC-A2, CBC-A3
# Create cluster
New-Cluster –Name CBC-A -Node CBC-A1, CBC-A2, CBC-A3 -StaticAddress 192.168.100.110
# Rename and configure networks
(Get-ClusterNetwork).Name = “Internal”
(Get-ClusterNetwork).Role = 3
(Get-Cluster).UseClientAccessNetworksForSharedVolumes=1
# Rename Witness Disk
$w = Get-ClusterResource | ? { $_.OwnerGroup -eq "Cluster Group" -and $_.ResourceType -eq "Physical Disk"}
$w.Name = "WitnessDisk"
# Add remaining disks to Cluster Shared Volumes
Get-ClusterResource | ? OwnerGroup -eq "Available Storage" | Add-ClusterSharedVolume
# Create Scale-Out File Server
Add-ClusterScaleOutFileServerRole CBC-F
# Create SMB shares
1..6 | % {
MD C:\ClusterStorage\Volume$_\Share
New-SmbShare -Name Share$_ -Path C:\ClusterStorage\Volume$_\Share -FullAccess CBC.Test\Administrator
Set-SmbPathAcl -ShareName Share$_
}
7) Configure CBC-V
Start-VM –name VM5
# Preparation steps: Install WS2012R2, rename computer
Rename-Computer -NewName CBC-V -Restart
# Install required roles and features, restarts at the end
Install-WindowsFeature RSAT-Clustering -IncludeAllSubFeature
# Configure Internal NIC with a static IP address for DNS / DC
Get-NetAdapter | Rename-NetAdapter -NewName Internal
Set-NetIPInterface -InterfaceAlias Internal -DHCP Disabled
Remove-NetIPAddress -InterfaceAlias Internal -Confirm:$false
New-NetIPAddress -InterfaceAlias Internal -IPAddress 192.168.100.104 -PrefixLength 24
Set-DnsClientServerAddress -InterfaceAlias Internal -ServerAddresses 192.168.100.100
# Join domain, restart the machine
Add-Computer -DomainName CBC.TEST -Credential (Get-Credential) -Restart
# Map the SMB shares
1..6 | % {
$d ="PQRSTU"[($_-1)] + “:”
New-SmbMapping -LocalPath $d -RemotePath \\CBC-F\Share$_ -Persistent $true
}
# Create the test files
1..6 | % {
$f ="PQRSTU"[($_-1)] + “:\testfile.dat”
fsutil file createnew $f (256MB)
fsutil file setvaliddata $f (256MB)
}
# Configure SMB Bandwidth Limits to avoid overloading the virtual environment
Add-WindowsFeature FS-SMBBW
Set-SmbBandwidthLimit -Category Default -BytesPerSecond 16MB
# Run SQLIO (assumes SQLIO.EXE was copied to C:\SLQIO)
c:\sqlio\sqlio.exe -s9999 -kR -t1 -o16 -b8 -BN -LS -frandom -dPQRSTU testfile.dat
# To remove a node while the workload is running (wait a few minutes for rebalancing)
Remove-ClusterNode -Cluster CBC-A –Name CBC-A3
# To add a node while the workload is running (wait a few minutes for rebalancing)
Add-ClusterNode -Cluster CBC-A –Name CBC-A3
8) Running the tests without VMM (from CBC-V)
Start Performance Monitor
Start a performance monitor session
Switch to “Histogram Bar” view to show the performance side-by-side
Add a counter for SMB Server Shares, Data bytes/sec, _total instance for CBC-A1, CBC-A2 and CBC-A3.
Query the cluster shared volume ownership on Cluster A, with 2 nodes
Get-ClusterSharedVolume -Cluster CBC-A | Sort OwnerNode | FT OwnerNode, Name,
State -AutoSize
OwnerNode Name
--------- ---CBC-A1
Cluster
CBC-A1
Cluster
CBC-A1
Cluster
CBC-A2
Cluster
CBC-A2
Cluster
CBC-A2
Cluster
Disk
Disk
Disk
Disk
Disk
Disk
6
3
4
7
2
5
State
----Online
Online
Online
Online
Online
Online
Run SQLIO to issue 8KB IOs
C:\sqlio\sqlio.exe -s9999 -kR -t1 -o16 -b8 -BN -LS -frandom -dPQRSTU testfile.dat
Add a 3rd node and wait for it to take 1/3 of the load
Add-ClusterNode -Cluster CBC-A -Name CBC-A3
Wait 2 and a half minutes to transition to the following state.
Re- query the cluster shared volume ownership on Cluster A, now with 3 nodes
Get-ClusterSharedVolume -Cluster CBC-A | Sort OwnerNode | FT OwnerNode, Name,
State -AutoSize
OwnerNode Name
--------- ---CBC-A1
Cluster
CBC-A1
Cluster
CBC-A2
Cluster
CBC-A2
Cluster
CBC-A3
Cluster
CBC-A3
Cluster
Disk
Disk
Disk
Disk
Disk
Disk
3
6
5
7
2
4
State
----Online
Online
Online
Online
Online
Online
9) Installing VMM on CBC-V
# Install .NET Framework
Install-WindowsFeature NET-Framework-Core
# Install Windows Server 2012 R2 ADK
C:\ADK\adksetup.exe
# Select only the “Deployment Tools” and “Windows PE” options
# Install SQL Server 2012 SP1
# New SQL Server standalone installation, Feature Selection: Database Engine Services
When prompted for feature selection, install ALL of the following:
Database Engine Services
Optionally – consider adding the following to ease administration:
Management Tools – Basic and Complete (for running queries and configuring SQL services)
On the Instance configuration, choose a default instance, or a named instance. Default instances are fine for
testing and labs. Production clustered instances of SQL will generally be a named instance. For the purposes of
the POC, choose default instance to keep things simple.
On the Server configuration screen, set SQL Server Agent to Automatic. Click “Use the same account for all SQL
Server Services, and input the SQL service account and password we created earlier.
On the Collation Tab – make sure SQL_Latin1_General_CP1_CI_AS is selected, as that is the ONLY collation
supported.
# Install VMM 2012 R2
# Features selected to be added: VMM management server, VMM console
# Database: VirtualManagerDB database will be created on CBCBDA-V
# Service Account: Local System account
10) Configuring VMM on CBC-V
Bring the File Server Cluster under VMM management
Select Fabric and use the option to add Storage Devices
Add a Windows-based file server
Specify the full path to the file server cluster:
Verify the File Server Cluster was properly discovered by VMM
Check the provider
Check the Storage Spaces discovery
Check the Scale-Out File Server and file share discovery
11) Verifying systems’ configuration
PS C:\> Get-PhysicalDisk -CimSession CBC-A1, CBC-A2, CBC-A3 | Sort
PSComputerName, Size
PS C:\> Get-Disk -CimSession CBC-A1, CBC-A2, CBC-A3 | Sort PSComputerName, Size
Number Friendly Name
OperationalStatus Total Size Partition Style
PSComputerName
------ ----------------------------- ---------- ---------------------------4
Microsoft Storage Space Device Online
1 GB
MBR
CBC-A1
10
Microsoft Storage Space Device Online
3 GB
MBR
CBC-A1
8
Microsoft Storage Space Device Online
3 GB
MBR
0
MBR
4
MBR
5
MBR
0
MBR
4
MBR
5
MBR
0
MBR
CBC-A1
Virtual HD ATA Device
CBC-A1
Microsoft Storage Space
CBC-A2
Microsoft Storage Space
CBC-A2
Virtual HD ATA Device
CBC-A2
Microsoft Storage Space
CBC-A3
Microsoft Storage Space
CBC-A3
Virtual HD ATA Device
CBC-A3
Online
40 GB
Device Online
3 GB
Device Online
3 GB
Online
40 GB
Device Online
3 GB
Device Online
3 GB
Online
40 GB
PS C:\> Get-ClusterNode -Cluster CBC-A | FT Cluster, NodeName, State, Id -AutoSize
Cluster NodeName State Id
------- -------- ----- -CBC-A CBC-A1
Up 2
CBC-A CBC-A2
Up 1
CBC-A CBC-A3
Up 3
PS C:\> Get-ClusterResource -Cluster CBC-A | FT –AutoSize
Name
State OwnerGroup
ResourceType
-------- --------------------Cluster IP Address
Online Cluster Group IP Address
Cluster Name
Online Cluster Group Network Name
Cluster Pool 1
Online Cluster Group Storage Pool
CBC-F
Online CBC-F
Distributed Network Name
Scale-Out File Server (\\CBC-F) Online CBC-F
Scale Out File Server
WitnessDisk
Online Cluster Group Physical Disk
PS C:\> Get-ClusterSharedVolume -Cluster CBC-A | FT -AutoSize
Name
---Cluster
Cluster
Cluster
Cluster
Cluster
Cluster
Disk
Disk
Disk
Disk
Disk
Disk
2
3
4
5
6
7
State
----Online
Online
Online
Online
Online
Online
Node
---CBC-A3
CBC-A2
CBC-A3
CBC-A1
CBC-A2
CBC-A1
PS C:\> Get-SmbShare Share* -CimSession CBC-F | FT -AutoSize
Name
---Share1
Share2
Share3
Share4
Share5
Share6
ScopeName Path
--------- ---CBC-F
C:\ClusterStorage\Volume1\Share
CBC-F
C:\ClusterStorage\Volume2\SHARE
CBC-F
C:\ClusterStorage\Volume3\Share
CBC-F
C:\ClusterStorage\Volume4\Share
CBC-F
C:\ClusterStorage\Volume5\SHARE
CBC-F
C:\ClusterStorage\Volume6\Share
Description PSComputerName
----------- -------------CBC-F
CBC-F
CBC-F
CBC-F
CBC-F
CBC-F