Got it working now! Here is the working code:
$vmhost = "myhostname"
$vDSName = "myvdsname"
$vmhostid = (get-vmhost $vmhost).extensiondata.Moref.value
$vDS = (Get-VDSwitch $vDSName)
$vDSView = Get-View -Id $vDS.id
$vDSView.UpdateViewData()
# ------- ReconfigureDvs_Task -------
$spec = New-Object VMware.Vim.DVSConfigSpec
$spec.configVersion = $vDSView.Config.ConfigVersion
$spec.host = New-Object VMware.Vim.DistributedVirtualSwitchHostMemberConfigSpec[] (1)
$spec.host[0] = New-Object VMware.Vim.DistributedVirtualSwitchHostMemberConfigSpec
$spec.host[0].operation = "remove"
$spec.host[0].host = New-Object VMware.Vim.ManagedObjectReference
$spec.host[0].host.type = "HostSystem"
$spec.host[0].host.Value = $vmhostid
$_this = Get-View -Id $vDS.id
$_this.ReconfigureDvs_Task($spec)
Thanks for all your help!