In case someone wanted to use the improved version here's the final one.

abstract class PackUriBuilder
{
    private static readonly Dictionary<UriKind, Func<PackUriBuilder, Uri>> UriFactories = new Dictionary<UriKind, System.Func<PackUriBuilder, Uri>>
    {
        [UriKind.Absolute] = builder => new Uri(builder.AbsolutePath, UriKind.Absolute),
        [UriKind.Relative] = builder => new Uri(builder.RelativePath, UriKind.Relative),
    };

    private static readonly Dictionary<UriKind, Func<PackUriBuilder, string>> StringFactories = new Dictionary<UriKind, System.Func<PackUriBuilder, string>>
    {
        [UriKind.Absolute] = builder => builder.AbsolutePath,
        [UriKind.Relative] = builder => builder.RelativePath,
    };

    private UriKind _uriKind;

    protected PackUriBuilder(string path)
    {
        if (!path.StartsWith("/"))
        {
            throw new ArgumentException(paramName: nameof(path), message: @"Path must start with ""/"".");
        }
        RelativePath = path;
        _uriKind = UriKind.Absolute;
    }

    public string Scheme => "pack";

    public string RelativePath { get; }

    public abstract string AbsolutePath { get; }

    public static implicit operator string (PackUriBuilder builder) => builder.ToString();

    public static implicit operator Uri(PackUriBuilder builder) => UriFactories[builder._uriKind](builder);

    public override string ToString() => StringFactories[_uriKind](this);   

    public Uri ToUri() => this;

    public PackUriBuilder Relative()
    {
        _uriKind = UriKind.Relative;
        return this;
    }

    public PackUriBuilder Absolute()
    {
        _uriKind = UriKind.Absolute;
        return this;
    }

    public static LocalAssemblyResourceFilePackUriBuilder LocalAssemblyResourceFile(string path)
    {
        return new LocalAssemblyResourceFilePackUriBuilder(path);
    }

    public static ReferencedAssemblyResourceFilePackUriBuilder ReferencedAssemblyResourceFile(string path, string assemblyShortName)
    {
        return new ReferencedAssemblyResourceFilePackUriBuilder(path, assemblyShortName);
    }

    public static ContentFilePackUriBuilder ContentFile(string path)
    {
        return new ContentFilePackUriBuilder(path);
    }

    public static SiteOfOriginPackUriBuilder SiteOfOrigin(string path)
    {
        return new SiteOfOriginPackUriBuilder(path);
    }
}

And the builders:

class LocalAssemblyResourceFilePackUriBuilder : PackUriBuilder
{
    public LocalAssemblyResourceFilePackUriBuilder(string path) : base(path) { }

    public override string AbsolutePath => $"{Scheme}://application:,,,{RelativePath}";
}

class ReferencedAssemblyResourceFilePackUriBuilder : PackUriBuilder
{
    private readonly string _assemblyShortName;
    private string _version;
    private string _publicKey;

    public ReferencedAssemblyResourceFilePackUriBuilder(string path, string assemblyShortName) : base(path)
    {
        _assemblyShortName = assemblyShortName;
    }

    private string VersionOrEmpty => string.IsNullOrEmpty(_version) ? string.Empty : $";{_version}";

    private string PublicKeyOrEmpty => string.IsNullOrEmpty(_publicKey) ? string.Empty : $";{_publicKey}";

    public ReferencedAssemblyResourceFilePackUriBuilder Version(string version)
    {
        _version = version;
        return this;
    }

    public ReferencedAssemblyResourceFilePackUriBuilder PublicKey(string publicKey)
    {
        _publicKey = publicKey;
        return this;
    }

    public override string AbsolutePath => $"{Scheme}://application:,,,/{_assemblyShortName}{VersionOrEmpty}{PublicKeyOrEmpty};component{RelativePath}";
}

class ContentFilePackUriBuilder : PackUriBuilder
{
    public ContentFilePackUriBuilder(string path) : base(path) { }

    public override string AbsolutePath => $"{Scheme}://application:,,,{RelativePath}";
}


class SiteOfOriginPackUriBuilder : PackUriBuilder
{
    public SiteOfOriginPackUriBuilder(string path) : base(path) { }

    public override string AbsolutePath => $"{Scheme}://siteoforigin:,,,{RelativePath}";
}