Remove the leftover geometry shader scaffolding

Nothing has generated or used a geometry shader since the GS paths were removed
- GeometryShaderGenerator is gone, and ShaderWriter's BeginGSMain/EndGSMain had
no callers at all. Removes ShaderStage::Geometry and everything hanging off it:
the GS preambles and GSMain helpers in ShaderWriter, the stage mappings in all
three thin3d backends, the D3D11 geometry shader plumbing (curGS_, the pipeline
and module members, gs_4_0 compilation), CreateGeometryShaderD3D11, the unused
PipelineFlags::USES_GEOMETRY_SHADER and PipelineManagerVulkan's
UsesGeometryShader().

Also stop enabling the Vulkan geometryShader device feature, since we no longer
have any use for it.

Kept on purpose: the device feature is still listed in the feature dumps (like
other capabilities we don't use), and the Vulkan shader cache header keeps its
now-always-zero geometry shader count so the on-disk format stays compatible.

Common/GPU/D3D11/thin3d_d3d11.cpp

@@ -248,7 +248,6 @@ private:
 	ComPtr<ID3D11InputLayout> curInputLayout_;
 	ComPtr<ID3D11VertexShader> curVS_;
 	ComPtr<ID3D11PixelShader> curPS_;
-	ComPtr<ID3D11GeometryShader> curGS_;
 	D3D11_PRIMITIVE_TOPOLOGY curTopology_ = D3D11_PRIMITIVE_TOPOLOGY_UNDEFINED;
 
 	ComPtr<ID3D11Buffer> nextVertexBuffer_;
@@ -847,7 +846,6 @@ public:
 
 	ComPtr<ID3D11VertexShader> vs;
 	ComPtr<ID3D11PixelShader> ps;
-	ComPtr<ID3D11GeometryShader> gs;
 };
 
 class D3D11Pipeline : public Pipeline {
@@ -868,7 +866,6 @@ public:
 
 	ComPtr<ID3D11VertexShader> vs;
 	ComPtr<ID3D11PixelShader> ps;
-	ComPtr<ID3D11GeometryShader> gs;
 	D3D11_PRIMITIVE_TOPOLOGY topology = D3D11_PRIMITIVE_TOPOLOGY_UNDEFINED;
 
 	std::vector<D3D11ShaderModule *> shaderModules;
@@ -1088,11 +1085,9 @@ ShaderModule *D3D11DrawContext::CreateShaderModule(ShaderStage stage, ShaderLang
 
 	const char *vertexModel = "vs_4_0";
 	const char *fragmentModel = "ps_4_0";
-	const char *geometryModel = "gs_4_0";
 	if (featureLevel_ <= D3D_FEATURE_LEVEL_9_3) {
 		vertexModel = "vs_4_0_level_9_1";
 		fragmentModel = "ps_4_0_level_9_1";
-		geometryModel = nullptr;
 	}
 
 	std::string compiled;
@@ -1101,11 +1096,6 @@ ShaderModule *D3D11DrawContext::CreateShaderModule(ShaderStage stage, ShaderLang
 	switch (stage) {
 	case ShaderStage::Fragment: target = fragmentModel; break;
 	case ShaderStage::Vertex: target = vertexModel; break;
-	case ShaderStage::Geometry:
-		if (!geometryModel)
-			return nullptr;
-		target = geometryModel;
-		break;
 	case ShaderStage::Compute:
 	default:
 		Crash();
@@ -1146,9 +1136,6 @@ ShaderModule *D3D11DrawContext::CreateShaderModule(ShaderStage stage, ShaderLang
 	case ShaderStage::Fragment:
 		result = device_->CreatePixelShader(data, dataSize, nullptr, &module->ps);
 		break;
-	case ShaderStage::Geometry:
-		result = device_->CreateGeometryShader(data, dataSize, nullptr, &module->gs);
-		break;
 	default:
 		ERROR_LOG(Log::G3D, "Unsupported shader stage");
 		result = S_FALSE;
@@ -1201,9 +1188,6 @@ Pipeline *D3D11DrawContext::CreateGraphicsPipeline(const PipelineDesc &desc, con
 		case ShaderStage::Fragment:
 			dPipeline->ps = module->ps;
 			break;
-		case ShaderStage::Geometry:
-			dPipeline->gs = module->gs;
-			break;
 		case ShaderStage::Compute:
 			break;
 		}
@@ -1247,7 +1231,6 @@ void D3D11DrawContext::Invalidate(InvalidationFlags flags) {
 		curRaster_ = nullptr;
 		curPS_.Reset();
 		curVS_.Reset();
-		curGS_.Reset();
 		curInputLayout_.Reset();
 		curTopology_ = D3D11_PRIMITIVE_TOPOLOGY_UNDEFINED;
 		curPipeline_= nullptr;
@@ -1290,10 +1273,6 @@ void D3D11DrawContext::ApplyCurrentState() {
 		context_->PSSetShader(curPipeline_->ps.Get(), nullptr, 0);
 		curPS_ = curPipeline_->ps;
 	}
-	if (curGS_ != curPipeline_->gs) {
-		context_->GSSetShader(curPipeline_->gs.Get(), nullptr, 0);
-		curGS_ = curPipeline_->gs;
-	}
 	if (curTopology_ != curPipeline_->topology) {
 		context_->IASetPrimitiveTopology(curPipeline_->topology);
 		curTopology_ = curPipeline_->topology;
@@ -1648,7 +1627,6 @@ void D3D11DrawContext::BeginFrame(DebugFlags debugFlags) {
 	context_->IASetInputLayout(curInputLayout_.Get());
 	context_->VSSetShader(curVS_.Get(), nullptr, 0);
 	context_->PSSetShader(curPS_.Get(), nullptr, 0);
-	context_->GSSetShader(curGS_.Get(), nullptr, 0);
 	if (curTopology_ != D3D11_PRIMITIVE_TOPOLOGY_UNDEFINED) {
 		context_->IASetPrimitiveTopology(curTopology_);
 	}

Common/GPU/OpenGL/thin3d_gl.cpp

@@ -201,7 +201,6 @@ GLuint ShaderStageToOpenGL(ShaderStage stage) {
 	case ShaderStage::Vertex: return GL_VERTEX_SHADER;
 #ifndef USING_GLES2
 	case ShaderStage::Compute: return GL_COMPUTE_SHADER;
-	case ShaderStage::Geometry: return GL_GEOMETRY_SHADER;
 #endif
 	case ShaderStage::Fragment:
 	default:

Common/GPU/Shader.cpp

@@ -20,7 +20,6 @@ const char *ShaderStageAsString(ShaderStage stage) {
 	switch (stage) {
 	case ShaderStage::Fragment: return "Fragment";
 	case ShaderStage::Vertex: return "Vertex";
-	case ShaderStage::Geometry: return "Geometry";
 	case ShaderStage::Compute: return "Compute";
 	default: return "(unknown)";
 	}

Common/GPU/Shader.h

@@ -29,7 +29,6 @@ const char *ShaderLanguageAsString(ShaderLanguage lang);
 enum class ShaderStage {
 	Vertex,
 	Fragment,
-	Geometry,
 	Compute,
 };
 

Common/GPU/ShaderTranslation.cpp

@@ -55,7 +55,6 @@
 static EShLanguage GetShLanguageFromStage(const ShaderStage stage) {
 	switch (stage) {
 	case ShaderStage::Vertex: return EShLangVertex;
-	case ShaderStage::Geometry: return EShLangGeometry;
 	case ShaderStage::Fragment: return EShLangFragment;
 	case ShaderStage::Compute: return EShLangCompute;
 	default: return EShLangVertex;

Common/GPU/ShaderWriter.cpp

@@ -52,32 +52,6 @@ static const char * const vulkan_glsl_preamble_vs =
 "precision highp float;\n"
 "\n";
 
-static const char * const hlsl_preamble_gs =
-"#define vec2 float2\n"
-"#define vec3 float3\n"
-"#define vec4 float4\n"
-"#define ivec2 int2\n"
-"#define ivec4 int4\n"
-"#define mat2 float2x2\n"
-"#define mat4 float4x4\n"
-"#define mat3x4 float4x3\n"  // note how the conventions are backwards
-"#define splat3(x) vec3(x, x, x)\n"
-"#define lowp\n"
-"#define mediump\n"
-"#define highp\n"
-"#define inversesqrt rsqrt\n"
-"#define floatBitsToUint asuint\n"
-"#define uintBitsToFloat asfloat\n"
-"\n";
-
-static const char * const vulkan_glsl_preamble_gs =
-"#extension GL_ARB_separate_shader_objects : enable\n"
-"#extension GL_ARB_shading_language_420pack : enable\n"
-"#define mul(x, y) ((x) * (y))\n"
-"#define splat3(x) vec3(x)\n"
-"precision highp float;\n"
-"\n";
-
 static const char * const hlsl_preamble_vs =
 "#define vec2 float2\n"
 "#define vec3 float3\n"
@@ -138,9 +112,6 @@ void ShaderWriter::Preamble(Slice<const char *> extensions) {
 		case ShaderStage::Fragment:
 			W(vulkan_glsl_preamble_fs);
 			break;
-		case ShaderStage::Geometry:
-			W(vulkan_glsl_preamble_gs);
-			break;
 		default:
 			break;
 		}
@@ -154,9 +125,6 @@ void ShaderWriter::Preamble(Slice<const char *> extensions) {
 			W(hlsl_preamble_fs);
 			W(hlsl_d3d11_preamble_fs);
 			break;
-		case ShaderStage::Geometry:
-			W(hlsl_preamble_gs);
-			break;
 		default:
 			break;
 		}
@@ -187,11 +155,6 @@ void ShaderWriter::Preamble(Slice<const char *> extensions) {
 			}
 			C("#define gl_VertexIndex gl_VertexID\n");
 			break;
-		case ShaderStage::Geometry:
-			if (lang_.gles) {
-				C("precision highp float;\n");
-			}
-			break;
 		default:
 			break;
 		}
@@ -340,43 +303,6 @@ void ShaderWriter::BeginFSMain(Slice<UniformDef> uniforms, Slice<VaryingDef> var
 	}
 }
 
-void ShaderWriter::BeginGSMain(Slice<VaryingDef> varyings, Slice<VaryingDef> outVaryings) {
-	_assert_(this->stage_ == ShaderStage::Geometry);
-	switch (lang_.shaderLanguage) {
-	case HLSL_D3D11:
-		// Untested, but should work.
-		C("\nstruct GS_OUTPUT {\n");
-		for (auto &varying : outVaryings) {
-			F("  %s %s : %s;\n", varying.type, varying.name, semanticNames[varying.semantic]);
-		}
-		F("  vec4 pos : %s;\n", lang_.shaderLanguage == HLSL_D3D11 ? "SV_Position" : "POSITION");
-		C("};\n");
-		C("#define EmitVertex() emit.Append(gsout)\n");
-
-		C("void main(");
-		for (auto &varying : varyings) {
-			F("  in %s %s : %s, ", varying.type, varying.name, semanticNames[varying.semantic]);
-		}
-		C("inout TriangleStream<GS_OUTPUT> emit) {\n");
-		C("  GS_OUTPUT gsout;\n");
-		break;
-	case GLSL_VULKAN:
-		for (auto &varying : varyings) {
-			F("layout(location = %d) %s in %s %s[];  // %s\n", varying.index, varying.precision ? varying.precision : "", varying.type, varying.name, semanticNames[varying.semantic]);
-		}
-		for (auto &varying : outVaryings) {
-			F("layout(location = %d) %s out %s %s;  // %s\n", varying.index, varying.precision ? varying.precision : "", varying.type, varying.name, semanticNames[varying.semantic]);
-		}
-		C("\nvoid main() {\n");
-		break;
-	case GLSL_3xx:
-		C("\nvoid main() {\n");
-		break;
-	default:
-		break;
-	}
-}
-
 void ShaderWriter::EndVSMain(Slice<VaryingDef> varyings) {
 	_assert_(this->stage_ == ShaderStage::Vertex);
 	switch (lang_.shaderLanguage) {
@@ -419,11 +345,6 @@ void ShaderWriter::EndFSMain(const char *vec4_color_variable) {
 	C("}\n");
 }
 
-void ShaderWriter::EndGSMain() {
-	_assert_(this->stage_ == ShaderStage::Geometry);
-	C("}\n");
-}
-
 void ShaderWriter::HighPrecisionFloat() {
 	if ((ShaderLanguageIsOpenGL(lang_.shaderLanguage) && lang_.gles) || lang_.shaderLanguage == GLSL_VULKAN) {
 		C("precision highp float;\n");

Common/GPU/ShaderWriter.h

@@ -97,12 +97,10 @@ public:
 	// Simple shaders with no special tricks.
 	void BeginVSMain(Slice<InputDef> inputs, Slice<UniformDef> uniforms, Slice<VaryingDef> varyings);
 	void BeginFSMain(Slice<UniformDef> uniforms, Slice<VaryingDef> varyings);
-	void BeginGSMain(Slice<VaryingDef> varyings, Slice<VaryingDef> outVaryings);
 
 	// For simple shaders that output a single color, we can deal with this generically.
 	void EndVSMain(Slice<VaryingDef> varyings);
 	void EndFSMain(const char *vec4_color_variable);
-	void EndGSMain();
 
 	const ShaderLanguageDesc &Lang() const {
 		return lang_;

Common/GPU/Vulkan/VulkanContext.cpp

@@ -822,7 +822,6 @@ VkResult VulkanContext::CreateDevice(int physical_device, const std::vector<cons
 	deviceFeatures_.enabled.standard.samplerAnisotropy = deviceFeatures_.available.standard.samplerAnisotropy;
 	deviceFeatures_.enabled.standard.shaderClipDistance = deviceFeatures_.available.standard.shaderClipDistance;
 	deviceFeatures_.enabled.standard.shaderCullDistance = deviceFeatures_.available.standard.shaderCullDistance;
-	deviceFeatures_.enabled.standard.geometryShader = deviceFeatures_.available.standard.geometryShader;
 	deviceFeatures_.enabled.standard.sampleRateShading = deviceFeatures_.available.standard.sampleRateShading;
 
 	// A host application (e.g. a libretro frontend) may require some additional features to be enabled.

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -47,7 +47,7 @@ enum class PipelineFlags : u8 {
 	NONE = 0,
 	USES_BLEND_CONSTANT = (1 << 1),
 	USES_DEPTH_STENCIL = (1 << 2),  // Reads or writes the depth or stencil buffers.
-	USES_GEOMETRY_SHADER = (1 << 3),
+	// Note: (1 << 3) is free, it used to be USES_GEOMETRY_SHADER.
 	USES_MULTIVIEW = (1 << 4),  // Inherited from the render pass it was created with.
 	USES_DISCARD = (1 << 5),
 	USES_FLAT_SHADING = (1 << 6),

Common/GPU/Vulkan/thin3d_vulkan.cpp

@@ -174,7 +174,6 @@ public:
 VkShaderStageFlagBits StageToVulkan(ShaderStage stage) {
 	switch (stage) {
 	case ShaderStage::Vertex: return VK_SHADER_STAGE_VERTEX_BIT;
-	case ShaderStage::Geometry: return VK_SHADER_STAGE_GEOMETRY_BIT;
 	case ShaderStage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT;
 	case ShaderStage::Fragment: return VK_SHADER_STAGE_FRAGMENT_BIT;
 	}

GPU/D3D11/D3D11Util.cpp

@@ -113,20 +113,3 @@ HRESULT CreateComputeShaderD3D11(ID3D11Device *device, const char *code, size_t
 
 	return device->CreateComputeShader(byteCode.data(), byteCode.size(), nullptr, ppComputeShader);
 }
-
-HRESULT CreateGeometryShaderD3D11(ID3D11Device *device, const char *code, size_t codeSize, D3D_FEATURE_LEVEL featureLevel, UINT flags, ID3D11GeometryShader **ppGeometryShader) {
-	if (ppGeometryShader)
-		*ppGeometryShader = nullptr;
-	if (featureLevel <= D3D_FEATURE_LEVEL_9_3)
-		return S_FALSE;
-	std::string errorMessage;
-	std::vector<uint8_t> byteCode = CompileShaderToBytecodeD3D11(code, codeSize, "gs_5_0", flags, &errorMessage);
-	if (byteCode.empty()) {
-		if (!errorMessage.empty()) {
-			ERROR_LOG(Log::G3D, "%s", errorMessage.c_str());
-		}
-		return S_FALSE;
-	}
-
-	return device->CreateGeometryShader(byteCode.data(), byteCode.size(), nullptr, ppGeometryShader);
-}

GPU/D3D11/D3D11Util.h

@@ -78,7 +78,6 @@ std::vector<uint8_t> CompileShaderToBytecodeD3D11(const char *code, size_t codeS
 HRESULT CreateVertexShaderD3D11(ID3D11Device *device, const char *code, size_t codeSize, std::vector<uint8_t> *byteCodeOut, D3D_FEATURE_LEVEL featureLevel, UINT flags, ID3D11VertexShader **);
 HRESULT CreatePixelShaderD3D11(ID3D11Device *device, const char *code, size_t codeSize, D3D_FEATURE_LEVEL featureLevel, UINT flags, ID3D11PixelShader **);
 HRESULT CreateComputeShaderD3D11(ID3D11Device *device, const char *code, size_t codeSize, D3D_FEATURE_LEVEL featureLevel, UINT flags, ID3D11ComputeShader **);
-HRESULT CreateGeometryShaderD3D11(ID3D11Device *device, const char *code, size_t codeSize, D3D_FEATURE_LEVEL featureLevel, UINT flags, ID3D11GeometryShader **);
 
 #define ASSERT_SUCCESS(x) \
 	if (!SUCCEEDED((x))) \

GPU/Vulkan/PipelineManagerVulkan.h

@@ -37,7 +37,6 @@ class VulkanRenderManager;
 class VulkanContext;
 class VulkanVertexShader;
 class VulkanFragmentShader;
-class VulkanGeometryShader;
 class ShaderManagerVulkan;
 class DrawEngineCommon;
 
@@ -73,7 +72,6 @@ struct VulkanPipeline {
 
 	bool UsesBlendConstant() const { return (pipelineFlags & PipelineFlags::USES_BLEND_CONSTANT) != 0; }
 	bool UsesDepthStencil() const { return (pipelineFlags & PipelineFlags::USES_DEPTH_STENCIL) != 0; }
-	bool UsesGeometryShader() const { return (pipelineFlags & PipelineFlags::USES_GEOMETRY_SHADER) != 0; }
 	bool UsesDiscard() const { return (pipelineFlags & PipelineFlags::USES_DISCARD) != 0; }
 	bool UsesFlatShading() const { return (pipelineFlags & PipelineFlags::USES_FLAT_SHADING) != 0; }
 

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -73,7 +73,6 @@ static Promise<VkShaderModule> *CompileShaderModuleAsync(VulkanContext *vulkan,
 				switch (stage) {
 				case VK_SHADER_STAGE_VERTEX_BIT: createTag = "game_vertex"; break;
 				case VK_SHADER_STAGE_FRAGMENT_BIT: createTag = "game_fragment"; break;
-				case VK_SHADER_STAGE_GEOMETRY_BIT: createTag = "game_geometry"; break;
 				case VK_SHADER_STAGE_COMPUTE_BIT: createTag = "game_compute"; break;
 				default: break;
 				}
@@ -378,7 +377,7 @@ struct VulkanCacheHeader {
 	uint32_t detectFlags;
 	int numVertexShaders;
 	int numFragmentShaders;
-	int numGeometryShaders;
+	int unused_numGeometryShaders;  // Always 0, kept so the file format stays compatible.
 };
 
 bool ShaderManagerVulkan::LoadCacheFlags(FILE *f, DrawEngineVulkan *drawEngine) {
@@ -464,7 +463,7 @@ bool ShaderManagerVulkan::LoadCache(FILE *f) {
 		}
 	}
 
-	NOTICE_LOG(Log::G3D, "ShaderCache: Loaded %d vertex, %d fragment shaders and %d geometry shaders (failed %d)", header.numVertexShaders, header.numFragmentShaders, header.numGeometryShaders, failCount);
+	NOTICE_LOG(Log::G3D, "ShaderCache: Loaded %d vertex and %d fragment shaders (failed %d)", header.numVertexShaders, header.numFragmentShaders, failCount);
 	return true;
 }
 
@@ -476,7 +475,7 @@ void ShaderManagerVulkan::SaveCache(FILE *f, DrawEngineVulkan *drawEngine) {
 	header.detectFlags = 0;
 	header.numVertexShaders = (int)vsCache_.size();
 	header.numFragmentShaders = (int)fsCache_.size();
-	header.numGeometryShaders = 0;
+	header.unused_numGeometryShaders = 0;
 	bool writeFailed = fwrite(&header, sizeof(header), 1, f) != 1;
 	vsCache_.Iterate([&](const VShaderID &id, VulkanVertexShader *vs) {
 		writeFailed = writeFailed || fwrite(&id, sizeof(id), 1, f) != 1;

unittest/TestShaderGenerators.cpp

@@ -94,7 +94,6 @@ bool GenerateVShader(VShaderID id, char *buffer, ShaderLanguage lang, Draw::Bugs
 static VkShaderStageFlagBits StageToVulkan(ShaderStage stage) {
 	switch (stage) {
 	case ShaderStage::Vertex: return VK_SHADER_STAGE_VERTEX_BIT;
-	case ShaderStage::Geometry: return VK_SHADER_STAGE_GEOMETRY_BIT;
 	case ShaderStage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT;
 	case ShaderStage::Fragment: return VK_SHADER_STAGE_FRAGMENT_BIT;
 	}
@@ -111,7 +110,6 @@ bool TestCompileShader(const char *buffer, ShaderLanguage lang, ShaderStage stag
 		switch (stage) {
 		case ShaderStage::Vertex: programType = "vs_4_0"; break;
 		case ShaderStage::Fragment: programType = "ps_4_0"; break;
-		case ShaderStage::Geometry: programType = "gs_4_0"; break;
 		default:
 			*errorMessage = "Unknown shader stage";
 			return false;