From a0c57e190ffdd4ebe70b9936acabd99a1b1c95de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= Date: Sat, 3 Oct 2026 10:17:16 -0600 Subject: [PATCH] SoftGPU: Pass a span's colors a channel at a time The span pixel function and the four-pixel bilinear sampler work in vector lanes, a channel per vector, but took and returned the colors a pixel at a time, so every span was transposed twice on the way. Now they take a channel at a time with a stride. The staged rows keep each chunk's colors that way from the start (their stage walks a channel at a time anyway), and DrawRectangle builds them so. Co-Authored-By: Claude Opus 5.5 (1M context) --- GPU/Software/DrawPixel.cpp | 23 +++---- GPU/Software/DrawPixel.h | 5 +- GPU/Software/Rasterizer.cpp | 118 ++++++++++++++++++++---------------- GPU/Software/Sampler.cpp | 27 ++++----- GPU/Software/Sampler.h | 5 +- 5 files changed, 95 insertions(+), 83 deletions(-) diff --git a/GPU/Software/DrawPixel.cpp b/GPU/Software/DrawPixel.cpp index 82ec6ec7e2..be8212ff2d 100644 --- a/GPU/Software/DrawPixel.cpp +++ b/GPU/Software/DrawPixel.cpp @@ -901,7 +901,7 @@ static inline Vec4S32 SpanStencilOp(GEStencilOp op, Vec4S32 old, Vec4S32 replace // The pixels of a span one per vector lane, as DrawPixelInline does them one at a time. Everything but logic // ops and the signed blend factors; false when the state needs those. template -static inline bool DrawSpanVector(int x, int y, const int *maskIn, const int *zIn, const int *fogIn, const Vec4 *colors, const PixelFuncID &pixelID) { +static inline bool DrawSpanVector(int x, int y, const int *maskIn, const int *zIn, const int *fogIn, const int *colors, int colorStride, const PixelFuncID &pixelID) { if constexpr (!clearMode) { if (pixelID.applyLogicOp) return false; @@ -918,15 +918,8 @@ static inline bool DrawSpanVector(int x, int y, const int *maskIn, const int *zI auto select = [](Vec4S32 m, Vec4S32 a, Vec4S32 b) { return (a & m) | b.AndNot(m); }; - alignas(16) int ch[4][4]; - for (int i = 0; i < 4; ++i) { - ch[0][i] = colors[i].x; - ch[1][i] = colors[i].y; - ch[2][i] = colors[i].z; - ch[3][i] = colors[i].w; - } - Vec4S32 r = clamp255(Vec4S32::Load(ch[0])), g = clamp255(Vec4S32::Load(ch[1])), b = clamp255(Vec4S32::Load(ch[2])); - const Vec4S32 a = clamp255(Vec4S32::Load(ch[3])); + Vec4S32 r = clamp255(Vec4S32::Load(colors)), g = clamp255(Vec4S32::Load(colors + colorStride)), b = clamp255(Vec4S32::Load(colors + 2 * colorStride)); + const Vec4S32 a = clamp255(Vec4S32::Load(colors + 3 * colorStride)); const Vec4S32 z = Vec4S32::Load(zIn); // -1 for the pixels that aren't drawn. Vec4S32 dead = Vec4S32::Load(maskIn).Shr<31>(); @@ -1142,12 +1135,14 @@ static inline bool DrawSpanVector(int x, int y, const int *maskIn, const int *zI } template -static void SOFTRAST_CALL DrawSpanPixels(int x, int y, const int *mask, const int *z, const int *fog, const Vec4 *colors, const PixelFuncID &pixelID) { - if (DrawSpanVector(x, y, mask, z, fog, colors, pixelID)) +static void SOFTRAST_CALL DrawSpanPixels(int x, int y, const int *mask, const int *z, const int *fog, const int *colors, int colorStride, const PixelFuncID &pixelID) { + if (DrawSpanVector(x, y, mask, z, fog, colors, colorStride, pixelID)) return; for (int i = 0; i < 4; ++i) { - if (mask[i] >= 0) - DrawPixelInline(x + i, y, z[i], fog[i], ToVec4IntArg(colors[i]), pixelID); + if (mask[i] >= 0) { + const Vec4 color(colors[i], colors[colorStride + i], colors[2 * colorStride + i], colors[3 * colorStride + i]); + DrawPixelInline(x + i, y, z[i], fog[i], ToVec4IntArg(color), pixelID); + } } } diff --git a/GPU/Software/DrawPixel.h b/GPU/Software/DrawPixel.h index f7f4bdab8c..0d758aa653 100644 --- a/GPU/Software/DrawPixel.h +++ b/GPU/Software/DrawPixel.h @@ -42,8 +42,9 @@ namespace Rasterizer { typedef void (SOFTRAST_CALL *SingleFunc)(int x, int y, int z, int fog, Vec4IntArg color_in, const PixelFuncID &pixelID); SingleFunc GetSingleFunc(const PixelFuncID &id, BinManager *binner); -// The pixels x to x + 3 of row y with mask[i] >= 0. z, fog and colors are per pixel. -typedef void (SOFTRAST_CALL *SpanFunc)(int x, int y, const int *mask, const int *z, const int *fog, const Math3D::Vec4 *colors, const PixelFuncID &pixelID); +// The pixels x to x + 3 of row y with mask[i] >= 0. z and fog are per pixel, colors a channel at a time: +// channel c of pixel i is colors[c * colorStride + i]. +typedef void (SOFTRAST_CALL *SpanFunc)(int x, int y, const int *mask, const int *z, const int *fog, const int *colors, int colorStride, const PixelFuncID &pixelID); // Only where single is the C++ one (as GetSingleFunc returned it), nullptr otherwise. SpanFunc GetSpanFunc(const PixelFuncID &id, SingleFunc single); diff --git a/GPU/Software/Rasterizer.cpp b/GPU/Software/Rasterizer.cpp index 12c2ae92f9..8c1cbacffa 100644 --- a/GPU/Software/Rasterizer.cpp +++ b/GPU/Software/Rasterizer.cpp @@ -702,7 +702,20 @@ static inline void CalculateSamplingParams(const float ds, const float dt, float // Four pixels in a row. ds and dt are the largest UV derivatives along x and y, for auto LOD without planes // (gpu/probe exp92, exact for affine mappings). sameQ: the pixels share their q, so also their level. -static inline void ApplyTexturing(const RasterizerState &state, Vec4 *prim_color, const Vec4 &mask, const Vec4 &s, const Vec4 &t, const Vec4 &q, float ds, float dt, bool sameQ, float autoGrad = -1.0f) { +// A span's colors a channel at a time (as SpanFunc takes them), from per pixel ones. +static inline void ColorsToLanes(const Vec4 in[4], int out[16]) { + for (int i = 0; i < 4; ++i) { + for (int c = 0; c < 4; ++c) + out[c * 4 + i] = in[i][c]; + } +} + +static inline Vec4 LaneColor(const int *colors, int stride, int i) { + return Vec4(colors[i], colors[stride + i], colors[2 * stride + i], colors[3 * stride + i]); +} + +// colors a channel at a time, as SpanFunc takes them. +static inline void ApplyTexturing(const RasterizerState &state, int *colors, int colorStride, const Vec4 &mask, const Vec4 &s, const Vec4 &t, const Vec4 &q, float ds, float dt, bool sameQ, float autoGrad = -1.0f) { int level; int levelFrac; bool bilinear; @@ -730,7 +743,7 @@ static inline void ApplyTexturing(const RasterizerState &state, Vec4 *prim_ active |= 1 << i; } if (allBilinear) { - state.linearQuad(s.AsArray(), t.AsArray(), levels, fracs, active, state.texptr, state.texbufw, prim_color, state.samplerID); + state.linearQuad(s.AsArray(), t.AsArray(), levels, fracs, active, state.texptr, state.texbufw, colors, colorStride, state.samplerID); return; } } @@ -738,7 +751,9 @@ static inline void ApplyTexturing(const RasterizerState &state, Vec4 *prim_ if (mask[i] >= 0) { if (perPixel) CalculateSamplingParams(ds, dt, q[i], state, level, levelFrac, bilinear, autoGrad); - prim_color[i] = ApplyTexturing(s[i], t[i], ToVec4IntArg(prim_color[i]), level, levelFrac, bilinear, state); + const Vec4 out = ApplyTexturing(s[i], t[i], ToVec4IntArg(LaneColor(colors, colorStride, i)), level, levelFrac, bilinear, state); + for (int c = 0; c < 4; ++c) + colors[c * colorStride + i] = out[c]; } } } @@ -1236,6 +1251,9 @@ static inline void GetTextureCoordinatesGE(const UVPlanes &planes, int64_t cente // The staged rows' last stage (DrawTriangleSlice): texturing, the secondary color and the pixels, span by // span. Specialized for what's fixed per triangle. +// Pixels per stage of a staged row. +static constexpr int STAGED_CHUNK = 64; + struct StagedSpans { const RasterizerState *state; const UVPlanes *uvPlanes; @@ -1250,7 +1268,7 @@ struct StagedSpans { }; template -static void DrawStagedSpans(const StagedSpans &ctx, int count, int chunkX, int64_t qv, int64_t sv, int64_t tv, int *maskBuf, int *zBuf, int *fogBuf, Vec4 *colorBuf, const Vec3 *secBuf) { +static void DrawStagedSpans(const StagedSpans &ctx, int count, int chunkX, int64_t qv, int64_t sv, int64_t tv, int *maskBuf, int *zBuf, int *fogBuf, int *colorBuf, const int *secBuf) { const RasterizerState &state = *ctx.state; const UVPlanes &uvPlanes = *ctx.uvPlanes; const int64_t qdx = uvPlanes.q.kx * SCREEN_SCALE_FACTOR, sdx = uvPlanes.s.kx * SCREEN_SCALE_FACTOR, tdx = uvPlanes.t.kx * SCREEN_SCALE_FACTOR; @@ -1258,7 +1276,7 @@ static void DrawStagedSpans(const StagedSpans &ctx, int count, int chunkX, int64 const Vec4 mask(maskBuf[k], maskBuf[k + 1], maskBuf[k + 2], maskBuf[k + 3]); if (!AnyMask(mask)) continue; - Vec4 *prim_color = &colorBuf[k]; + int *prim_color = &colorBuf[k]; const int spanPX = (chunkX + k) & 0x3FF; if constexpr (textured) { Vec4 s, t, q; @@ -1283,17 +1301,19 @@ static void DrawStagedSpans(const StagedSpans &ctx, int count, int chunkX, int64 s *= 1.0f / (float)(1 << state.samplerID.width0Shift); t *= 1.0f / (float)(1 << state.samplerID.height0Shift); } - ApplyTexturing(state, prim_color, mask, s, t, q, 0.0f, 0.0f, ctx.sameQ, ctx.autoGrad); + ApplyTexturing(state, prim_color, STAGED_CHUNK, mask, s, t, q, 0.0f, 0.0f, ctx.sameQ, ctx.autoGrad); } if (ctx.addSecondary) { - for (int i = 0; i < 4; ++i) - prim_color[i] += Vec4(secBuf[k + i], 0); + for (int c = 0; c < 3; ++c) { + int *channel = prim_color + c * STAGED_CHUNK; + (Vec4S32::Load(channel) + Vec4S32::Load(secBuf + c * STAGED_CHUNK + k)).Store(channel); + } } - state.drawSpan(spanPX, ctx.y, &maskBuf[k], &zBuf[k], &fogBuf[k], prim_color, state.pixelID); + state.drawSpan(spanPX, ctx.y, &maskBuf[k], &zBuf[k], &fogBuf[k], prim_color, STAGED_CHUNK, state.pixelID); } } -typedef void (*StagedSpansFunc)(const StagedSpans &ctx, int count, int chunkX, int64_t qv, int64_t sv, int64_t tv, int *maskBuf, int *zBuf, int *fogBuf, Vec4 *colorBuf, const Vec3 *secBuf); +typedef void (*StagedSpansFunc)(const StagedSpans &ctx, int count, int chunkX, int64_t qv, int64_t sv, int64_t tv, int *maskBuf, int *zBuf, int *fogBuf, int *colorBuf, const int *secBuf); template static StagedSpansFunc PickStagedSpans(bool lodQ, bool through) { @@ -1501,10 +1521,11 @@ void DrawTriangleSlice( // The usual case, a row at a time in stages: coverage, depth and the early depth test, then colors and // fog for all its pixels, then texturing and the pixels span by span. Each stage keeps little state. if (staged) { - constexpr int CHUNK = 64; + constexpr int CHUNK = STAGED_CHUNK; alignas(16) int maskBuf[CHUNK], zBuf[CHUNK], fogBuf[CHUNK]; - Vec4 colorBuf[CHUNK]; - Vec3 secBuf[CHUNK]; + // A channel at a time. + alignas(16) int colorBuf[4][CHUNK]; + alignas(16) int secBuf[3][CHUNK]; int chunkX = p.x; for (int64_t chunk = firstSpan; chunk <= hi; chunk += CHUNK, chunkX = (chunkX + CHUNK) & 0x3FF) { const int count = (int)std::min(CHUNK, ((hi - chunk) / 4 + 1) * 4); @@ -1549,23 +1570,22 @@ void DrawTriangleSlice( } } - if (!flatColor0) { - for (int c = 0; c < 4; ++c) - walk(color0Planes[c], [&](int k, int64_t v) { colorBuf[k][c] = std::clamp((int)v, 0, 255); }); - } else { - for (int k = 0; k < count; ++k) - colorBuf[k] = v2_c0; + for (int c = 0; c < 4; ++c) { + if (!flatColor0) { + walk(color0Planes[c], [&](int k, int64_t v) { colorBuf[c][k] = std::clamp((int)v, 0, 255); }); + } else { + for (int k = 0; k < count; ++k) + colorBuf[c][k] = v2_c0[c]; + } } - if (!flatColor1) { - for (int c = 0; c < 3; ++c) - walk(color1Planes[c], [&](int k, int64_t v) { secBuf[k][c] = std::clamp((int)v, 0, 255); }); - } else { - for (int k = 0; k < count; ++k) - secBuf[k] = v2_c1; - } - if (DoubleSecondaryColor(state)) { - for (int k = 0; k < count; ++k) - secBuf[k] = secBuf[k] + secBuf[k]; + const int secScale = DoubleSecondaryColor(state) ? 2 : 1; + for (int c = 0; c < 3; ++c) { + if (!flatColor1) { + walk(color1Planes[c], [&](int k, int64_t v) { secBuf[c][k] = std::clamp((int)v, 0, 255) * secScale; }); + } else { + for (int k = 0; k < count; ++k) + secBuf[c][k] = v2_c1[c] * secScale; + } } if (!noFog) { // The 8-bit fog of each vertex through the depth plane, like Gouraud color (gpu/probe exp21). @@ -1584,7 +1604,7 @@ void DrawTriangleSlice( stagedCtx.coverLo = coverLo; stagedCtx.coverHi = coverHi; stagedCtx.y = p.y; - drawStagedSpans(stagedCtx, count, chunkX, qv, sv, tv, maskBuf, zBuf, fogBuf, colorBuf, secBuf); + drawStagedSpans(stagedCtx, count, chunkX, qv, sv, tv, maskBuf, zBuf, fogBuf, colorBuf[0], secBuf[0]); } continue; } @@ -1745,7 +1765,11 @@ void DrawTriangleSlice( q = Vec4::AssignToAll(1.0f / clipw); } // A span shares its q, except for the planes' per-pixel q when it doesn't matter. - ApplyTexturing(state, prim_color, mask, s, t, q, ds, dt, !uvPlanes.valid || lodUsesQ, autoGrad); + alignas(16) int lanes[16]; + ColorsToLanes(prim_color, lanes); + ApplyTexturing(state, lanes, 4, mask, s, t, q, ds, dt, !uvPlanes.valid || lodUsesQ, autoGrad); + for (int i = 0; i < 4; ++i) + prim_color[i] = LaneColor(lanes, 4, i); } } @@ -1774,7 +1798,9 @@ void DrawTriangleSlice( PROFILE_THIS_SCOPE("draw_tri_px"); #if !defined(SOFTGPU_MEMORY_TAGGING_DETAILED) if (state.drawSpan) { - state.drawSpan(p.x, p.y, mask.AsArray(), z.AsArray(), fog.AsArray(), prim_color, pixelID); + alignas(16) int lanes[16]; + ColorsToLanes(prim_color, lanes); + state.drawSpan(p.x, p.y, mask.AsArray(), z.AsArray(), fog.AsArray(), lanes, 4, pixelID); continue; } #endif @@ -1926,10 +1952,10 @@ void DrawRectangle(const VertexData &v0, const VertexData &v1, const BinCoords & p.x = (p.x + 4) & 0x3FF) { Vec4 mask = scissor_mask; - Vec4 prim_color[4]; - for (int i = 0; i < 4; ++i) { - prim_color[i] = c0; - } + // A channel at a time, as SpanFunc takes them. + alignas(16) int prim_color[16]; + for (int c = 0; c < 4; ++c) + Vec4S32::Splat(c0[c]).Store(prim_color + c * 4); if (state.pixelID.earlyZChecks) { for (int i = 0; i < 4; ++i) { @@ -1947,29 +1973,19 @@ void DrawRectangle(const VertexData &v0, const VertexData &v1, const BinCoords & // Pixel centers are at 16k + 7 here, the GE's at 16k + 8. A sprite always has planes, so the LOD // comes from their gradient and q. GetTextureCoordinatesGE(uvPlanes, curX + 1, curY + 1, s, t, q); - ApplyTexturing(state, prim_color, mask, s, t, q, 0.0f, 0.0f, false, autoGrad); + ApplyTexturing(state, prim_color, 4, mask, s, t, q, 0.0f, 0.0f, false, autoGrad); } if (!state.pixelID.clearMode) { - for (int i = 0; i < 4; ++i) { -#if defined(_M_SSE) - // TODO: Tried making Vec4 do this, but things got slower. - const __m128i sec = _mm_and_si128(sec_color.ivec, _mm_set_epi32(0, -1, -1, -1)); - prim_color[i].ivec = _mm_add_epi32(prim_color[i].ivec, sec); -#elif PPSSPP_ARCH(ARM64_NEON) - int32x4_t sec = vsetq_lane_s32(0, sec_color.ivec, 3); - prim_color[i].ivec = vaddq_s32(prim_color[i].ivec, sec); -#else - prim_color[i] += Vec4(sec_color, 0); -#endif - } + for (int c = 0; c < 3; ++c) + (Vec4S32::Load(prim_color + c * 4) + Vec4S32::Splat(sec_color[c])).Store(prim_color + c * 4); } PROFILE_THIS_SCOPE("draw_rect_px"); #if !defined(SOFTGPU_MEMORY_TAGGING_DETAILED) if (state.drawSpan) { Vec4 fogs = fog, zs = z; - state.drawSpan(p.x, p.y, mask.AsArray(), zs.AsArray(), fogs.AsArray(), prim_color, state.pixelID); + state.drawSpan(p.x, p.y, mask.AsArray(), zs.AsArray(), fogs.AsArray(), prim_color, 4, state.pixelID); continue; } #endif @@ -1980,7 +1996,7 @@ void DrawRectangle(const VertexData &v0, const VertexData &v1, const BinCoords & } subp.x = p.x + i; - state.drawPixel(subp.x, subp.y, z[i], fog[i], ToVec4IntArg(prim_color[i]), state.pixelID); + state.drawPixel(subp.x, subp.y, z[i], fog[i], ToVec4IntArg(LaneColor(prim_color, 4, i)), state.pixelID); #if defined(SOFTGPU_MEMORY_TAGGING_DETAILED) uint32_t row = gstate.getFrameBufAddress() + subp.y * state.pixelID.cached.framebufStride * bpp; diff --git a/GPU/Software/Sampler.cpp b/GPU/Software/Sampler.cpp index e78d28c5ea..216c7219a4 100644 --- a/GPU/Software/Sampler.cpp +++ b/GPU/Software/Sampler.cpp @@ -1073,7 +1073,7 @@ static inline void TextureFunction4(const Vec4S32 prim[4], const Vec4S32 tex[4], } template -static void SOFTRAST_CALL SampleLinearQuadT(const float *s, const float *t, const int *level, const int *levelFrac, int active, const u8 *const *texptr, const uint16_t *texbufw, Vec4 *colors, const SamplerID &samplerID) { +static void SOFTRAST_CALL SampleLinearQuadT(const float *s, const float *t, const int *level, const int *levelFrac, int active, const u8 *const *texptr, const uint16_t *texbufw, int *colors, int colorStride, const SamplerID &samplerID) { if (!active) return; uint64_t c[4]; @@ -1095,25 +1095,21 @@ static void SOFTRAST_CALL SampleLinearQuadT(const float *s, const float *t, cons c[i] = LerpSpread(c[i], c1[i], levelFrac[first]); } // The texture function, in lanes too. - alignas(16) int lanes[2][4][4]; + alignas(16) int texLanes[4][4]; for (int i = 0; i < 4; ++i) { - for (int ch = 0; ch < 4; ++ch) { - lanes[0][ch][i] = colors[i][ch]; - lanes[1][ch][i] = (int)((c[i] >> (16 * ch)) & 0xFFFF); - } + for (int ch = 0; ch < 4; ++ch) + texLanes[ch][i] = (int)((c[i] >> (16 * ch)) & 0xFFFF); } Vec4S32 prim[4], tex[4], out[4]; for (int ch = 0; ch < 4; ++ch) { - prim[ch] = Vec4S32::Load(lanes[0][ch]); - tex[ch] = Vec4S32::Load(lanes[1][ch]); + prim[ch] = Vec4S32::Load(colors + ch * colorStride); + tex[ch] = Vec4S32::Load(texLanes[ch]); } TextureFunction4(prim, tex, samplerID, out); + alignas(16) static const int laneBits[4] = { 1, 2, 4, 8 }; + const Vec4S32 keep = (Vec4S32::Splat(active) & Vec4S32::LoadAligned(laneBits)).CompareEq(Vec4S32::Zero()); for (int ch = 0; ch < 4; ++ch) - out[ch].Store(lanes[0][ch]); - for (int i = 0; i < 4; ++i) { - if (active & (1 << i)) - colors[i] = Vec4(lanes[0][0][i], lanes[0][1][i], lanes[0][2][i], lanes[0][3][i]); - } + (out[ch].AndNot(keep) | (prim[ch] & keep)).Store(colors + ch * colorStride); return; } @@ -1130,7 +1126,10 @@ static void SOFTRAST_CALL SampleLinearQuadT(const float *s, const float *t, cons if (!(active & (1 << i))) continue; const Vec4 texcolor((int)(c[i] & 0xFFFF), (int)((c[i] >> 16) & 0xFFFF), (int)((c[i] >> 32) & 0xFFFF), (int)(c[i] >> 48)); - colors[i] = GetTextureFunctionOutput(ToVec4IntArg(colors[i]), ToVec4IntArg(texcolor), samplerID); + const Vec4 prim(colors[i], colors[colorStride + i], colors[2 * colorStride + i], colors[3 * colorStride + i]); + const Vec4 out = GetTextureFunctionOutput(ToVec4IntArg(prim), ToVec4IntArg(texcolor), samplerID); + for (int ch = 0; ch < 4; ++ch) + colors[ch * colorStride + i] = out[ch]; } } diff --git a/GPU/Software/Sampler.h b/GPU/Software/Sampler.h index 63f335ba49..6e77dfe034 100644 --- a/GPU/Software/Sampler.h +++ b/GPU/Software/Sampler.h @@ -47,8 +47,9 @@ typedef Rasterizer::Vec4IntResult (SOFTRAST_CALL *LinearFunc)(float s, float t, LinearFunc GetLinearFunc(SamplerID id, BinManager *binner); // Bilinear samples for the pixels of a quad with bit i of active set, each at its own level, through the -// texture function: colors goes in as the primitive's colors and comes out textured. -typedef void (SOFTRAST_CALL *LinearQuadFunc)(const float *s, const float *t, const int *level, const int *levelFrac, int active, const u8 *const *texptr, const uint16_t *texbufw, Math3D::Vec4 *colors, const SamplerID &samplerID); +// texture function: colors goes in as the primitive's colors and comes out textured. A channel at a time, +// as for SpanFunc. +typedef void (SOFTRAST_CALL *LinearQuadFunc)(const float *s, const float *t, const int *level, const int *levelFrac, int active, const u8 *const *texptr, const uint16_t *texbufw, int *colors, int colorStride, const SamplerID &samplerID); // Only for the samplers without a JIT (linear is what GetLinearFunc returned), nullptr otherwise. LinearQuadFunc GetLinearQuadFunc(const SamplerID &id, LinearFunc linear);