Add a sanity check for shader linking, active in debug mode

GPU/Directx9/ShaderManagerDX9.cpp

@@ -514,7 +514,7 @@ VSShader *ShaderManagerDX9::ApplyShader(int prim, u32 vertType) {
 	bool useHWTransform = CanUseHardwareTransformDX9(prim);
 
 	VertexShaderIDDX9 VSID;
-	ComputeVertexShaderIDDX9(&VSID, vertType, prim, useHWTransform);
+	ComputeVertexShaderIDDX9(&VSID, vertType, useHWTransform);
 	FragmentShaderIDDX9 FSID;
 	ComputeFragmentShaderIDDX9(&FSID);
 

GPU/Directx9/VertexShaderGeneratorDX9.cpp

@@ -43,7 +43,7 @@ bool CanUseHardwareTransformDX9(int prim) {
 }
 
 // prim so we can special case for RECTANGLES :(
-void ComputeVertexShaderIDDX9(VertexShaderIDDX9 *id, u32 vertType, int prim, bool useHWTransform) {
+void ComputeVertexShaderIDDX9(VertexShaderIDDX9 *id, u32 vertType, bool useHWTransform) {
 	bool doTexture = gstate.isTextureMapEnabled() && !gstate.isModeClear();
 	bool doTextureProjection = gstate.getUVGenMode() == GE_TEXMAP_TEXTURE_MATRIX;
 	bool doShadeMapping = gstate.getUVGenMode() == GE_TEXMAP_ENVIRONMENT_MAP;

GPU/Directx9/VertexShaderGeneratorDX9.h

@@ -52,7 +52,7 @@ struct VertexShaderIDDX9
 
 bool CanUseHardwareTransformDX9(int prim);
 
-void ComputeVertexShaderIDDX9(VertexShaderIDDX9 *id, u32 vertType, int prim, bool useHWTransform);
+void ComputeVertexShaderIDDX9(VertexShaderIDDX9 *id, u32 vertType, bool useHWTransform);
 void GenerateVertexShaderDX9(int prim, char *buffer, bool useHWTransform);
 
 // Collapse to less skinning shaders to reduce shader switching, which is expensive.

GPU/GLES/ShaderManager.cpp

@@ -40,7 +40,7 @@
 #include "UI/OnScreenDisplay.h"
 #include "Framebuffer.h"
 
-Shader::Shader(const char *code, uint32_t shaderType, bool useHWTransform) : failed_(false), useHWTransform_(useHWTransform) {
+Shader::Shader(const char *code, uint32_t shaderType, bool useHWTransform, const ShaderID &shaderID) : failed_(false), useHWTransform_(useHWTransform), id_(shaderID) {
 	source_ = code;
 #ifdef SHADERLOG
 	OutputDebugStringUTF8(code);
@@ -714,6 +714,23 @@ void ShaderManager::DirtyLastShader() { // disables vertex arrays
 	lastShader_ = 0;
 }
 
+// This is to be used when debugging why incompatible shaders are being linked, like is
+// happening as I write this in Tactics Ogre
+bool ShaderManager::DebugAreShadersCompatibleForLinking(Shader *vs, Shader *fs) {
+	// Check clear mode flag just for starters.
+	ShaderID vsid = vs->ID();
+	ShaderID fsid = fs->ID();
+
+	// TODO: Make the flag fields more similar?
+	// Check DoTexture
+	if (((vsid.d[0] >> 4) & 1) != ((fsid.d[0] >> 1) & 1)) {
+		ERROR_LOG(G3D, "Texture enable flag mismatch!");
+		return false;
+	}
+
+	return true;
+}
+
 Shader *ShaderManager::ApplyVertexShader(int prim, u32 vertType) {
 	// This doesn't work - we miss some events that really do need to dirty the prescale.
 	// like changing the texmapmode.
@@ -730,12 +747,12 @@ Shader *ShaderManager::ApplyVertexShader(int prim, u32 vertType) {
 	bool useHWTransform = CanUseHardwareTransform(prim);
 
 	ShaderID VSID;
-	ComputeVertexShaderID(&VSID, vertType, prim, useHWTransform);
+	ComputeVertexShaderID(&VSID, vertType, useHWTransform);
 
 	// Just update uniforms if this is the same shader as last time.
 	if (lastShader_ != 0 && VSID == lastVSID_) {
 		lastVShaderSame_ = true;
-		return lastShader_->vs_;	// Already all set.
+		return lastShader_->vs_;  	// Already all set.
 	} else {
 		lastVShaderSame_ = false;
 	}
@@ -747,7 +764,7 @@ Shader *ShaderManager::ApplyVertexShader(int prim, u32 vertType) {
 	if (vsIter == vsCache_.end())	{
 		// Vertex shader not in cache. Let's compile it.
 		GenerateVertexShader(prim, vertType, codeBuffer_, useHWTransform);
-		vs = new Shader(codeBuffer_, GL_VERTEX_SHADER, useHWTransform);
+		vs = new Shader(codeBuffer_, GL_VERTEX_SHADER, useHWTransform, VSID);
 
 		if (vs->Failed()) {
 			ERROR_LOG(G3D, "Shader compilation failed, falling back to software transform");
@@ -760,7 +777,7 @@ Shader *ShaderManager::ApplyVertexShader(int prim, u32 vertType) {
 
 			// Can still work with software transform.
 			GenerateVertexShader(prim, vertType, codeBuffer_, false);
-			vs = new Shader(codeBuffer_, GL_VERTEX_SHADER, false);
+			vs = new Shader(codeBuffer_, GL_VERTEX_SHADER, false, VSID);
 		}
 
 		vsCache_[VSID] = vs;
@@ -785,7 +802,7 @@ LinkedShader *ShaderManager::ApplyFragmentShader(Shader *vs, int prim, u32 vertT
 	if (fsIter == fsCache_.end())	{
 		// Fragment shader not in cache. Let's compile it.
 		GenerateFragmentShader(codeBuffer_);
-		fs = new Shader(codeBuffer_, GL_FRAGMENT_SHADER, vs->UseHWTransform());
+		fs = new Shader(codeBuffer_, GL_FRAGMENT_SHADER, vs->UseHWTransform(), FSID);
 		fsCache_[FSID] = fs;
 	} else {
 		fs = fsIter->second;
@@ -805,6 +822,12 @@ LinkedShader *ShaderManager::ApplyFragmentShader(Shader *vs, int prim, u32 vertT
 	shaderSwitchDirty_ = 0;
 
 	if (ls == NULL) {
+		// Check if we can link these.
+#ifdef _DEBUG
+		if (!DebugAreShadersCompatibleForLinking(vs, fs)) {
+			return NULL;
+		}
+#endif
 		ls = new LinkedShader(vs, fs, vertType, vs->UseHWTransform(), lastShader_);  // This does "use" automatically
 		const LinkedShaderCacheEntry entry(vs, fs, ls);
 		linkedShaderCache_.push_back(entry);

GPU/GLES/ShaderManager.h

@@ -176,16 +176,18 @@ enum {
 
 class Shader {
 public:
-	Shader(const char *code, uint32_t shaderType, bool useHWTransform);
+	Shader(const char *code, uint32_t shaderType, bool useHWTransform, const ShaderID &shaderID);
 	~Shader();
 	uint32_t shader;
 	const std::string &source() const { return source_; }
 
 	bool Failed() const { return failed_; }
 	bool UseHWTransform() const { return useHWTransform_; }
+	const ShaderID &ID() const { return id_; }
 
 private:
 	std::string source_;
+	ShaderID id_;
 	bool failed_;
 	bool useHWTransform_;
 };
@@ -214,6 +216,7 @@ public:
 
 private:
 	void Clear();
+	static bool DebugAreShadersCompatibleForLinking(Shader *vs, Shader *fs);
 
 	struct LinkedShaderCacheEntry {
 		LinkedShaderCacheEntry(Shader *vs_, Shader *fs_, LinkedShader *ls_)

GPU/GLES/VertexShaderGenerator.cpp

@@ -48,8 +48,7 @@ bool CanUseHardwareTransform(int prim) {
 	return !gstate.isModeThrough() && prim != GE_PRIM_RECTANGLES;
 }
 
-// prim so we can special case for RECTANGLES :(
-void ComputeVertexShaderID(ShaderID *id, u32 vertType, int prim, bool useHWTransform) {
+void ComputeVertexShaderID(ShaderID *id, u32 vertType, bool useHWTransform) {
 	bool doTexture = gstate.isTextureMapEnabled() && !gstate.isModeClear();
 	bool doTextureProjection = gstate.getUVGenMode() == GE_TEXMAP_TEXTURE_MATRIX;
 	bool doShadeMapping = gstate.getUVGenMode() == GE_TEXMAP_ENVIRONMENT_MAP;

GPU/GLES/VertexShaderGenerator.h

@@ -25,5 +25,5 @@ struct ShaderID;
 
 bool CanUseHardwareTransform(int prim);
 
-void ComputeVertexShaderID(ShaderID *id, u32 vertexType, int prim, bool useHWTransform);
+void ComputeVertexShaderID(ShaderID *id, u32 vertexType, bool useHWTransform);
 void GenerateVertexShader(int prim, u32 vertexType, char *buffer, bool useHWTransform);