diff --git a/src/functions/private/Base64/ConvertFrom-Base64.ps1 b/src/functions/private/Base64/ConvertFrom-Base64.ps1
new file mode 100644
index 0000000..29434f1
--- /dev/null
+++ b/src/functions/private/Base64/ConvertFrom-Base64.ps1
@@ -0,0 +1,24 @@
+function ConvertFrom-Base64 {
+    <#
+        .SYNOPSIS
+        Converts a Base64 encoded string to a string.
+
+        .DESCRIPTION
+        Converts a Base64 encoded string to a string.
+
+        .EXAMPLE
+        ConvertFrom-Base64 -Base64String 'VGhpc0lzQU5pY2VTdHJpbmc='
+        ThisIsANiceString
+
+        Converts the Base64 encoded string 'VGhpc0lzQU5pY2VTdHJpbmc=' to a string.
+    #>
+    [OutputType([string])]
+    [CmdletBinding()]
+    param(
+        # The Base64 encoded string to convert.
+        [Parameter(Mandatory = $true)]
+        [string] $Base64String
+    )
+
+    [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($Base64String))
+}
diff --git a/src/functions/private/Base64/ConvertTo-Base64.ps1 b/src/functions/private/Base64/ConvertTo-Base64.ps1
new file mode 100644
index 0000000..ea646c9
--- /dev/null
+++ b/src/functions/private/Base64/ConvertTo-Base64.ps1
@@ -0,0 +1,23 @@
+function ConvertTo-Base64 {
+    <#
+        .SYNOPSIS
+        Converts a string to a Base64 encoded string.
+
+        .DESCRIPTION
+        Converts a string to a Base64 encoded string.
+
+        .EXAMPLE
+        ConvertTo-Base64 -String 'ThisIsANiceString'
+        VGhpc0lzQU5pY2VTdHJpbmc=
+
+        Converts the string 'ThisIsANiceString' to a Base64 encoded string.
+    #>
+    [OutputType([string])]
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [string] $String
+    )
+
+    [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($String))
+}
diff --git a/src/functions/private/Base64/Test-Base64.ps1 b/src/functions/private/Base64/Test-Base64.ps1
new file mode 100644
index 0000000..dc10f08
--- /dev/null
+++ b/src/functions/private/Base64/Test-Base64.ps1
@@ -0,0 +1,26 @@
+function Test-Base64 {
+    <#
+        .SYNOPSIS
+        Test if a string is a valid Base64 string.
+
+        .DESCRIPTION
+        Test if a string is a valid Base64 string.
+
+        .EXAMPLE
+        Test-Base64 -Base64String 'U29tZSBkYXRh'
+        True
+
+        Returns $true as the string is a valid Base64 string.
+    #>
+    [OutputType([bool])]
+    [CmdletBinding()]
+    param (
+        [string] $Base64String
+    )
+    try {
+        $null = [Convert]::FromBase64String($Base64String)
+        return $true
+    } catch {
+        return $false
+    }
+}
diff --git a/src/functions/private/Get-ContextInfo.ps1 b/src/functions/private/Get-ContextInfo.ps1
new file mode 100644
index 0000000..426fa63
--- /dev/null
+++ b/src/functions/private/Get-ContextInfo.ps1
@@ -0,0 +1,35 @@
+function Get-ContextInfo {
+    <#
+        .SYNOPSIS
+        Retrieves all context info from the context vault.
+
+        .DESCRIPTION
+        Retrieves all context info from the context vault.
+
+        .EXAMPLE
+        Get-ContextInfo
+
+        Get all context info from the context vault.
+    #>
+    param()
+
+    $vaultName = $script:Config.VaultName
+    $secretPrefix = $script:Config.SecretPrefix
+
+    Write-Verbose "Retrieving all context info from [$vaultName]"
+
+    Get-SecretInfo -Vault $vaultName | Where-Object { ($_.Name).StartsWith($secretPrefix) } | ForEach-Object {
+        $name64 = $_.Name -replace "^$secretPrefix"
+        if (Test-Base64 -Base64String $name64) {
+            $name = ConvertFrom-Base64 -Base64String $name64
+            Write-Verbose " + $name ($name64)"
+            [pscustomobject]@{
+                Name64     = $name64
+                SecretName = $_.Name
+                Name       = $name
+                Metadata   = $_.Metadata
+                Type       = $_.Type
+            }
+        }
+    }
+}
diff --git a/src/functions/public/Context/Get-Context.ps1 b/src/functions/public/Context/Get-Context.ps1
index e74690f..dfc7d67 100644
--- a/src/functions/public/Context/Get-Context.ps1
+++ b/src/functions/public/Context/Get-Context.ps1
@@ -34,32 +34,27 @@ filter Get-Context {
         Write-Debug "[$commandName] - Start"
         $null = Get-ContextVault
         $vaultName = $script:Config.VaultName
-        $secretPrefix = $script:Config.SecretPrefix
-        $fullID = "$secretPrefix$ID"
+        $contextInfos = Get-ContextInfo
     }
 
     process {
         try {
             if (-not $PSBoundParameters.ContainsKey('ID')) {
                 Write-Verbose "Retrieving all contexts from [$vaultName]"
-                $contexts = Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$secretPrefix*" }
             } elseif ([string]::IsNullOrEmpty($ID)) {
                 Write-Verbose "Return 0 contexts from [$vaultName]"
                 return
             } elseif ($ID.Contains('*')) {
-                # If wildcards are used, we can use the -Name parameter to filter the results. Its using the -like operator internally in the module.
-                Write-Verbose "Retrieving contexts matching [$ID] from [$vaultName]"
-                $contexts = Get-SecretInfo -Vault $vaultName -Name $fullID
+                Write-Verbose "Retrieving contexts like [$ID] from [$vaultName]"
+                $contextInfos = $contextInfos | Where-Object { $_.Name -like $ID }
             } else {
-                # Needs to use Where-Object in order to support special characters, like `[` and `]`.
                 Write-Verbose "Retrieving context [$ID] from [$vaultName]"
-                $contexts = Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -eq $fullID }
+                $contextInfos = $contextInfos | Where-Object { $_.Name -eq $ID }
             }
 
-            Write-Verbose "Found [$($contexts.Count)] contexts in [$vaultName]"
-            $contexts | ForEach-Object {
-                Write-Verbose " - $($_.Name)"
-                $contextJson = $_ | Get-Secret -AsPlainText
+            Write-Verbose "Found [$($contextInfos.Count)] contexts in [$vaultName]"
+            $contextInfos | ForEach-Object {
+                $contextJson = Get-Secret -Name $_.SecretName -Vault $vaultName -AsPlainText
                 ConvertFrom-ContextJson -JsonString $contextJson
             }
         } catch {
@@ -77,9 +72,8 @@ Register-ArgumentCompleter -CommandName Get-Context -ParameterName ID -ScriptBlo
     param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
     $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
 
-    Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$($script:Config.SecretPrefix)$wordToComplete*" } | ForEach-Object {
-        $Name = $_.Name -replace "^$($script:Config.SecretPrefix)"
-        [System.Management.Automation.CompletionResult]::new($Name, $Name, 'ParameterValue', $Name)
-    }
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
 }
-
diff --git a/src/functions/public/Context/Remove-Context.ps1 b/src/functions/public/Context/Remove-Context.ps1
index fffb82c..2fadfa0 100644
--- a/src/functions/public/Context/Remove-Context.ps1
+++ b/src/functions/public/Context/Remove-Context.ps1
@@ -34,16 +34,15 @@ filter Remove-Context {
         $commandName = $MyInvocation.MyCommand.Name
         Write-Debug "[$commandName] - Start"
         $null = Get-ContextVault
-        $vaultName = $script:Config.VaultName
-        $secretPrefix = $script:Config.SecretPrefix
-        $fullID = "$secretPrefix$ID"
     }
 
     process {
         try {
 
-            if ($PSCmdlet.ShouldProcess($fullID, 'Remove secret')) {
-                Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -eq $fullID } | Remove-Secret
+            if ($PSCmdlet.ShouldProcess($ID, 'Remove secret')) {
+                Get-ContextInfo | Where-Object { $_.Name -eq $ID } | ForEach-Object {
+                    Remove-Secret -Name $_.SecretName -Vault $script:Config.VaultName
+                }
             }
         } catch {
             Write-Error $_
@@ -60,8 +59,8 @@ Register-ArgumentCompleter -CommandName Remove-Context -ParameterName ID -Script
     param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
     $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
 
-    Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$($script:Config.SecretPrefix)$wordToComplete*" } | ForEach-Object {
-        $Name = $_.Name -replace "^$($script:Config.SecretPrefix)"
-        [System.Management.Automation.CompletionResult]::new($Name, $Name, 'ParameterValue', $Name)
-    }
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
 }
diff --git a/src/functions/public/Context/Rename-Context.ps1 b/src/functions/public/Context/Rename-Context.ps1
index 4c75517..56ad51a 100644
--- a/src/functions/public/Context/Rename-Context.ps1
+++ b/src/functions/public/Context/Rename-Context.ps1
@@ -61,3 +61,13 @@
         Write-Debug "[$commandName] - End"
     }
 }
+
+Register-ArgumentCompleter -CommandName Rename-Context -ParameterName ID -ScriptBlock {
+    param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
+    $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
+
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
+}
diff --git a/src/functions/public/Context/Set-Context.ps1 b/src/functions/public/Context/Set-Context.ps1
index 8994424..f30e4f8 100644
--- a/src/functions/public/Context/Set-Context.ps1
+++ b/src/functions/public/Context/Set-Context.ps1
@@ -37,26 +37,32 @@ function Set-Context {
         $null = Get-ContextVault
         $vaultName = $script:Config.VaultName
         $secretPrefix = $script:Config.SecretPrefix
-        $fullID = "$secretPrefix$ID"
     }
 
     process {
         try {
-            $secret = ConvertTo-ContextJson -Context $Context -ID $fullID
+            $secret = ConvertTo-ContextJson -Context $Context -ID $ID
         } catch {
             Write-Error $_
             throw 'Failed to convert context to JSON'
         }
 
+        try {
+            $name64 = ConvertTo-Base64 -String $ID
+        } catch {
+            Write-Error $_
+            throw 'Failed to convert ID to Base64'
+        }
+
         $param = @{
-            Name   = $fullID
+            Name   = "$secretPrefix$name64"
             Secret = $secret
             Vault  = $vaultName
         }
         Write-Verbose ($param | ConvertTo-Json -Depth 5)
 
         try {
-            if ($PSCmdlet.ShouldProcess($fullID, 'Set Secret')) {
+            if ($PSCmdlet.ShouldProcess($ID, 'Set Secret')) {
                 Set-Secret @param
             }
         } catch {
diff --git a/src/functions/public/ContextSetting/Get-ContextSetting.ps1 b/src/functions/public/ContextSetting/Get-ContextSetting.ps1
index baf5b3b..21726b3 100644
--- a/src/functions/public/ContextSetting/Get-ContextSetting.ps1
+++ b/src/functions/public/ContextSetting/Get-ContextSetting.ps1
@@ -57,10 +57,10 @@ Register-ArgumentCompleter -CommandName Get-ContextSetting -ParameterName ID -Sc
     param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
     $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
 
-    Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$($script:Config.SecretPrefix)$wordToComplete*" } | ForEach-Object {
-        $Name = $_.Name -replace "^$($script:Config.SecretPrefix)"
-        [System.Management.Automation.CompletionResult]::new($Name, $Name, 'ParameterValue', $Name)
-    }
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
 }
 
 Register-ArgumentCompleter -CommandName Get-ContextSetting -ParameterName Name -ScriptBlock {
diff --git a/src/functions/public/ContextSetting/Remove-ContextSetting.ps1 b/src/functions/public/ContextSetting/Remove-ContextSetting.ps1
index 578e120..a327fd9 100644
--- a/src/functions/public/ContextSetting/Remove-ContextSetting.ps1
+++ b/src/functions/public/ContextSetting/Remove-ContextSetting.ps1
@@ -78,10 +78,10 @@ Register-ArgumentCompleter -CommandName Remove-ContextSetting -ParameterName ID
     param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
     $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
 
-    Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$($script:Config.SecretPrefix)$wordToComplete*" } | ForEach-Object {
-        $Name = $_.Name -replace "^$($script:Config.SecretPrefix)"
-        [System.Management.Automation.CompletionResult]::new($Name, $Name, 'ParameterValue', $Name)
-    }
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
 }
 
 Register-ArgumentCompleter -CommandName Get-ContextSetting -ParameterName Name -ScriptBlock {
diff --git a/src/functions/public/ContextSetting/Set-ContextSetting.ps1 b/src/functions/public/ContextSetting/Set-ContextSetting.ps1
index 1163c42..822cee7 100644
--- a/src/functions/public/ContextSetting/Set-ContextSetting.ps1
+++ b/src/functions/public/ContextSetting/Set-ContextSetting.ps1
@@ -76,8 +76,8 @@ Register-ArgumentCompleter -CommandName Get-ContextSetting -ParameterName ID -Sc
     param($commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter)
     $null = $commandName, $parameterName, $wordToComplete, $commandAst, $fakeBoundParameter
 
-    Get-SecretInfo -Vault $vaultName | Where-Object { $_.Name -like "$($script:Config.SecretPrefix)$wordToComplete*" } | ForEach-Object {
-        $Name = $_.Name -replace "^$($script:Config.SecretPrefix)"
-        [System.Management.Automation.CompletionResult]::new($Name, $Name, 'ParameterValue', $Name)
-    }
+    Get-ContextInfo | Where-Object { $_.Name -like "$wordToComplete*" } |
+        ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_.Name, $_.Name, 'ParameterValue', $_.Name)
+        }
 }
diff --git a/tests/Context.Tests.ps1 b/tests/Context.Tests.ps1
index ed3c1a1..3a64481 100644
--- a/tests/Context.Tests.ps1
+++ b/tests/Context.Tests.ps1
@@ -119,7 +119,7 @@ Describe 'Context' {
             $result = Get-Context -ID 'TestID'
             $result | Should -Not -BeNullOrEmpty
             $result.Name | Should -Be 'TestName'
-            $result.ID | Should -Be 'Context:TestID'
+            $result.ID | Should -Be 'TestID'
         }
         It 'Set-Context -Context $Context - Context can hold a bigger object' {
             $Context = @{
@@ -134,7 +134,7 @@ Describe 'Context' {
             $result.Count | Should -Be 1
             $result | Should -Not -BeNullOrEmpty
             $result.AccessToken | Should -Be 'MySecret'
-            $result.ID | Should -Be 'Context:TestID2'
+            $result.ID | Should -Be 'TestID2'
         }
         It 'Set-Context -Context $Context - Context can be saved multiple times' {
             $Context = @{
@@ -150,7 +150,7 @@ Describe 'Context' {
             $result | Should -Not -BeNullOrEmpty
             $result.AccessToken | Should -Be 'MySecret'
             $result.RefreshToken | Should -Be 'MyRefreshedSecret'
-            $result.ID | Should -Be 'Context:TestID3'
+            $result.ID | Should -Be 'TestID3'
         }
     }
 
@@ -262,11 +262,11 @@ Describe 'Context' {
             }
 
             # Test to see if it can be run multiple times
-            Set-Context -Context $githubLoginContext -ID 'BigComplexObject'
-            Set-Context -Context $githubLoginContext -ID 'BigComplexObject'
-            Set-Context -Context $githubLoginContext -ID 'BigComplexObject'
-            Write-Verbose (Get-Secret -Name 'Context:BigComplexObject' -AsPlainText) -Verbose
-            $object = Get-Context -ID 'BigComplexObject'
+            Set-Context -Context $githubLoginContext -ID 'BigComplexObjectWith[specialchars]'
+            Set-Context -Context $githubLoginContext -ID 'BigComplexObjectWith[specialchars]'
+            Set-Context -Context $githubLoginContext -ID 'BigComplexObjectWith[specialchars]'
+            Write-Verbose (Get-Context -ID 'BigComplexObjectWith[specialchars]') -Verbose
+            $object = Get-Context -ID 'BigComplexObjectWith[specialchars]'
             $object.ApiRateLimits.Remaining | Should -Be 4985
             $object.AuthToken | Should -BeOfType [System.Security.SecureString]
             $object.AuthToken | ConvertFrom-SecureString -AsPlainText | Should -Be 'ghp_12345ABCDE67890FGHIJ'