NVIDIA's New FXAA Antialiasing Technology @ [H]

/*============================================================================
   FXAA3 QUALITY - PC
 	NVIDIA FXAA III.8 by TIMOTHY LOTTES
============================================================================*/

   #define FXAA_LINEAR 0
   #define FXAA_QUALITY__EDGE_THRESHOLD (1.0/16.0)
   #define FXAA_QUALITY__EDGE_THRESHOLD_MIN (1.0/16.0)
   #define FXAA_QUALITY__SUBPIX_CAP (3.0/4.0)
   #define FXAA_QUALITY__SUBPIX_TRIM (1.0/4.0)
   #define FXAA_QUALITY__SUBPIX_TRIM_SCALE  (1.0/(1.0 - FXAA_QUALITY__SUBPIX_TRIM))
   #define FXAA_SEARCH_STEPS     8
   #define FXAA_SEARCH_THRESHOLD (1.0/4.0)

float4 FxaaPixelShader(VS_OUTPUT_POST IN, float2 vPos : VPOS) : COLOR
{   

#define FxaaTexTop(t, p) tex2Dlod(t, float4(p, 0.0, 0.0))
#define FxaaTexOff(t, p, o, r) tex2Dlod(t, float4(p + (o * r), 0, 0))

float2 pos = IN.txcoord.xy;

float2 rcpFrame = float2(1/ScreenSize, ScreenScaleY/ScreenSize);
float4 rcpFrameOpt = float4(2/ScreenSize, 2*ScreenScaleY/ScreenSize, 0.5/ScreenSize, 0.5*ScreenScaleY/ScreenSize);

float lumaN = dot(FxaaTexOff(SamplerColor, pos.xy, float2(0, -1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1, 0), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));


float4 rgbyM;
rgbyM.xyz = FxaaTexTop(SamplerColor, pos.xy).xyz;
rgbyM.w = dot(rgbyM.xyz, float3(0.299, 0.587, 0.114));
float lumaE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1, 0), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaS = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 0, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaM = rgbyM.w;


float rangeMin = min(lumaM, min(min(lumaN, lumaW), min(lumaS, lumaE)));
float rangeMax = max(lumaM, max(max(lumaN, lumaW), max(lumaS, lumaE)));
float range = rangeMax - rangeMin;

if(range < max(FXAA_QUALITY__EDGE_THRESHOLD_MIN, rangeMax * FXAA_QUALITY__EDGE_THRESHOLD)) return rgbyM;


float lumaNW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1,-1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaNE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1,-1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaSW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaSE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));



float lumaL = (lumaN + lumaW + lumaE + lumaS) * 0.25;
float rangeL = abs(lumaL - lumaM);
float blendL = saturate((rangeL / range) - FXAA_QUALITY__SUBPIX_TRIM) * FXAA_QUALITY__SUBPIX_TRIM_SCALE; 
blendL = min(FXAA_QUALITY__SUBPIX_CAP, blendL);

float edgeVert = abs(lumaNW + (-2.0 * lumaN) + lumaNE) + 2.0 * abs(lumaW  + (-2.0 * lumaM) + lumaE ) + abs(lumaSW + (-2.0 * lumaS) + lumaSE);
   	float edgeHorz = abs(lumaNW + (-2.0 * lumaW) + lumaSW) + 2.0 * abs(lumaN  + (-2.0 * lumaM) + lumaS ) + abs(lumaNE + (-2.0 * lumaE) + lumaSE);
   	bool horzSpan = edgeHorz >= edgeVert;

float lengthSign = horzSpan ? -rcpFrame.y : -rcpFrame.x;
if(!horzSpan) lumaN = lumaW;
if(!horzSpan) lumaS = lumaE;
float gradientN = abs(lumaN - lumaM);
float gradientS = abs(lumaS - lumaM);
lumaN = (lumaN + lumaM) * 0.5;
lumaS = (lumaS + lumaM) * 0.5;

bool pairN = gradientN >= gradientS;
if(!pairN) lumaN = lumaS;
if(!pairN) gradientN = gradientS;
if(!pairN) lengthSign *= -1.0;
float2 posN;
posN.x = pos.x + (horzSpan ? 0.0 : lengthSign * 0.5);
posN.y = pos.y + (horzSpan ? lengthSign * 0.5 : 0.0);


gradientN *= FXAA_SEARCH_THRESHOLD;

float2 posP = posN;
float2 offNP = horzSpan ? 
float2(rcpFrame.x, 0.0) :
float2(0.0f, rcpFrame.y); 
float lumaEndN;
float lumaEndP;	
bool doneN = false;
bool doneP = false;
posN += offNP * (-1.5);
posP += offNP * ( 1.5);
for(int i = 0; i < FXAA_SEARCH_STEPS; i++) 
{
 lumaEndN = dot(FxaaTexTop(SamplerColor, posN.xy).xyz, float3(0.299, 0.587, 0.114));
 lumaEndP = dot(FxaaTexTop(SamplerColor, posP.xy).xyz, float3(0.299, 0.587, 0.114));
 bool doneN2 = abs(lumaEndN - lumaN) >= gradientN;
 bool doneP2 = abs(lumaEndP - lumaN) >= gradientN;
 if(doneN2 && !doneN) posN += offNP;
 if(doneP2 && !doneP) posP -= offNP;
 if(doneN2 && doneP2) break;
 doneN = doneN2;
 doneP = doneP2;
 if(!doneN) posN -= offNP * 2.0;
 if(!doneP) posP += offNP * 2.0; 
}

float dstN = horzSpan ? pos.x - posN.x : pos.y - posN.y;
float dstP = horzSpan ? posP.x - pos.x : posP.y - pos.y;

bool directionN = dstN < dstP;
lumaEndN = directionN ? lumaEndN : lumaEndP;

if(((lumaM - lumaN) < 0.0) == ((lumaEndN - lumaN) < 0.0)) 
lengthSign = 0.0;

float spanLength = (dstP + dstN);
dstN = directionN ? dstN : dstP;
float subPixelOffset = 0.5 + (dstN * (-1.0/spanLength));
subPixelOffset += blendL * (1.0/8.0);
subPixelOffset *= lengthSign;
float3 rgbF = FxaaTexTop(SamplerColor, float2(pos.x + (horzSpan ? 0.0 : subPixelOffset), pos.y + (horzSpan ? subPixelOffset : 0.0))).xyz;

   #if (FXAA_LINEAR == 1)
lumaL *= lumaL;
   #endif
   float lumaF = dot(rgbF, float3(0.299, 0.587, 0.114)) + (1.0/(65536.0*256.0));
   float lumaB = lerp(lumaF, lumaL, blendL);
   float scale = min(4.0, lumaB/lumaF);
   rgbF *= scale;

   return float4(rgbF, lumaM); 
}