GS:MTL: Add workaround for Apple GPU hardware bug

Author: TellowKrinkle

pcsx2/GS/Renderers/Metal/GSDeviceMTL.mm

@@ -1073,9 +1073,10 @@ static bool ConvertShaderNotNeeded(ShaderConvert shader)
 
 	// Init metal stuff
 	m_fn_constants = MRCTransfer([MTLFunctionConstantValues new]);
-	setFnConstantB(m_fn_constants, m_features.framebuffer_fetch,    GSMTLConstantIndex_FRAMEBUFFER_FETCH);
-	setFnConstantB(m_fn_constants, m_features.depth_feedback,       GSMTLConstantIndex_DEPTH_FEEDBACK);
-	setFnConstantB(m_fn_constants, m_dev.features.rov_requires_r32, GSMTLConstantIndex_ROV_NEEDS_R32);
+	setFnConstantB(m_fn_constants, m_features.framebuffer_fetch,       GSMTLConstantIndex_FRAMEBUFFER_FETCH);
+	setFnConstantB(m_fn_constants, m_features.depth_feedback,          GSMTLConstantIndex_DEPTH_FEEDBACK);
+	setFnConstantB(m_fn_constants, m_dev.features.rov_requires_r32,    GSMTLConstantIndex_ROV_NEEDS_R32);
+	setFnConstantB(m_fn_constants, m_dev.features.broken_shader_depth, GSMTLConstantIndex_BROKEN_SHADER_DEPTH);
 
 	m_draw_sync_fence = MRCTransfer([m_dev.dev newFence]);
 	[m_draw_sync_fence setLabel:@"Draw Sync Fence"];
@@ -2350,6 +2351,9 @@ static bool usesStencil(GSHWDrawConfig::DestinationAlphaMode dstalpha)
 	if (config.tex && (config.ds == config.tex || config.rt == config.tex))
 		EndRenderPass(); // Barrier
 
+	if (m_dev.features.broken_shader_depth && (config.depth.ztst >= ZTST_GEQUAL || config.depth.zwe))
+		config.ps.zfloor = true; // Depth must always go through shader (see tfx vs for comment with details)
+
 	size_t vertsize = config.nverts * sizeof(*config.verts);
 	size_t idxsize = config.vs.UseFixedExpandIndexBuffer() ? 0 : (config.nindices * sizeof(*config.indices));
 	Map allocation = Allocate(m_vertex_upload_buf, vertsize + idxsize);

pcsx2/GS/Renderers/Metal/GSMTLDeviceInfo.h

@@ -37,6 +37,7 @@ struct GSMTLDevice
 		bool rov                    : 1;
 		bool rov_requires_rt        : 1;
 		bool rov_requires_r32       : 1;
+		bool broken_shader_depth    : 1;
 		MetalVersion shader_version;
 		int max_texsize;
 	};

pcsx2/GS/Renderers/Metal/GSMTLDeviceInfo.mm

@@ -207,7 +207,7 @@ static DetectionResult detectIntelGPU(id<MTLDevice> dev, id<MTLLibrary> lib)
 	}
 	else if ([name containsString:@"Apple"])
 	{
-		// No special settings
+		features.broken_shader_depth = true;
 	}
 	else
 	{
@@ -228,6 +228,9 @@ static DetectionResult detectIntelGPU(id<MTLDevice> dev, id<MTLLibrary> lib)
 	if (char* env = getenv("MTL_ROV_WITH_RT"))
 		features.rov_requires_rt = env[0] == '1' || env[0] == 'y' || env[0] == 'Y';
 
+	if (char* env = getenv("MTL_SHADER_DEPTH_WORKAROUND"))
+		features.broken_shader_depth = env[0] == '1' || env[0] == 'y' || env[0] == 'Y';
+
 	features.max_texsize = GetMaxTextureSize(dev);
 
 	this->dev = std::move(dev);

pcsx2/GS/Renderers/Metal/GSMTLSharedHeader.h

@@ -166,6 +166,7 @@ enum GSMTLFnConstants
 	GSMTLConstantIndex_FRAMEBUFFER_FETCH,
 	GSMTLConstantIndex_DEPTH_FEEDBACK,
 	GSMTLConstantIndex_ROV_NEEDS_R32,
+	GSMTLConstantIndex_BROKEN_SHADER_DEPTH,
 	GSMTLConstantIndex_FST,
 	GSMTLConstantIndex_IIP,
 	GSMTLConstantIndex_VS_POINT_SIZE,

pcsx2/GS/Renderers/Metal/tfx.metal

@@ -14,6 +14,7 @@ constant uint SHUFFLE_READWRITE = 3;
 constant bool HAS_FBFETCH           [[function_constant(GSMTLConstantIndex_FRAMEBUFFER_FETCH)]];
 constant bool DEPTH_FEEDBACK        [[function_constant(GSMTLConstantIndex_DEPTH_FEEDBACK)]];
 constant bool ROV_NEEDS_R32         [[function_constant(GSMTLConstantIndex_ROV_NEEDS_R32)]];
+constant bool BROKEN_SHADER_DEPTH   [[function_constant(GSMTLConstantIndex_BROKEN_SHADER_DEPTH)]];
 constant bool FST                   [[function_constant(GSMTLConstantIndex_FST)]];
 constant bool IIP                   [[function_constant(GSMTLConstantIndex_IIP)]];
 constant bool VS_POINT_SIZE         [[function_constant(GSMTLConstantIndex_VS_POINT_SIZE)]];
@@ -248,6 +249,14 @@ static MainVSOut vs_main_run(thread const MainVSIn& v, constant GSMTLMainVSUnifo
 	if (VS_POINT_SIZE)
 		out.point_size = cb.point_size.x;
 
+	// Apple GPUs use slightly different algorithms to calculate the Z they send to the shader vs the Z they use in hardware.
+	// This breaks a lot of things (the most common is conservative depth rejecting pixels that should have depth equal to current depth but now don't).
+	// Work around by always routing depth through the shader, and never using hardware depth values.
+	// To allow us to continue to use [[depth(less)]] optimizations, add a bit in the VS and subtract it off in the FS,
+	// so that "equal" depth doesn't ever fail a hardware depth test.
+	if (BROKEN_SHADER_DEPTH)
+		out.p.z += exp_min32;
+
 	return out;
 }
 
@@ -1494,6 +1503,8 @@ struct PSMain
 	{
 		MainResult out = {};
 		float input_z = in.p.z;
+		if (BROKEN_SHADER_DEPTH)
+			input_z -= 0x1p-32;
 		if (PS_ZFLOOR)
 			input_z = floor(input_z * 0x1p32) * 0x1p-32;