FXAA in effect.txt doesn't load

[legacyy]

It's about the v0.163 for GTA IV.

I, and some other people, have the problem that the FXAA inside the effect.txt doesn't get loaded any more.

We figured it out that it's related to these 2 values here:
float ScreenSize.x; //width of the display resolution (1920 f.e.)
float ScreenScale.z; //screen proportions (1.333 for 1920/1080)

It seems like those aren't assigned anymore, so the compilation of the file fails.

Here's the effect.txt I'm talking about:
http://pastebin.com/9GT75wQc

Can you tell how the variables are called now?

[Marty McFly]

I suppose it's problem with the new ENB effect file standard. Quoted from enbdev effect section:

ScreenSize single float replaced by four float vector, where X component is the same width as before, Y is 1/width, Z is width/height, W is height/width.


what means the former values were:

float ScreenSize.x which was screen sitze in horizontal
float ScreenScale.z which is screen ratio, x size / y size

now it is much simpler. it has to be float4 ScreenSize whose

x component is e.g. 1920
y component is (here) 1/1920
z is former ScreenScale.z, x size / y size
and w is inverted z, y size / x size

what means you have to look through the shader and find the places where it wants to access the screensizes. for full HD screen it would be:

float4 ScreenSize = float4(1920,1/1920,1.777,0.5625);

but while writing that I think that the old values should work...because ScreenSize.x and z. stayed...wtf

[ENBSeries]

Compare effects from the site and do the same (they have two copies, old and new standarts).

[Андрей Раицкий]

Legacy , I think, that effect wiil be able to work with new dll's:

//++++++++++++++++++++++++++++++++++++++++++++
// ENBSeries effect file
// visit http://enbdev.com for updates
// Copyright (c) 2007-2011 Boris Vorontsov
//++++++++++++++++++++++++++++++++++++++++++++
//THIS IS HLSL FILE FORMAT FOR EXECUTING ADDITIONAL
//POST PROCESSING EFFECTS. MAKE THE COPY BEFORE CHANGING IT!


//enable blurring, useless, disabled at all
//#define EBLURRING

//enable sharpening
#define ESHARPENING

//if defined, color sharpen, otherwise sharp by gray
#define ESHARPENINGCOLOR

//enable noise in dark areas
#define ENOISE


float ShiftSamplingRange=0.0;
float SamplingRange=1.0; //sharpening or blurring range
float SharpeningAmount=0.5;
float ScanLineAmount=0.0;
float ScanLineRepeat=0.0; //0.5, 0.3333, 0.25, 0.125, so on
float NoiseAmount=0.0;

//+++++++++++++++++++++++++++++
//external parameters, do not modify
//+++++++++++++++++++++++++++++
//keyboard controlled temporary variables (in some versions exists in the config file). Press and hold key 1,2,3...8 together with PageUp or PageDown to modify. By default all set to 1.0
float4 tempF1; //0,1,2,3
float4 tempF2; //5,6,7,8
float4 tempF3; //9,0
//x=generic timer in range 0..1, period of 16777216 ms (4.6 hours), w=frame time elapsed (in seconds)
float4 Timer;
//x=Width, y=1/Width, z=ScreenScaleY, w=1/ScreenScaleY
float4 ScreenSize;

//textures
texture2D texColor;
texture2D texNoise;

sampler2D SamplerColor = sampler_state
{
Texture = <texColor>;
MinFilter = LINEAR;
MagFilter = LINEAR;
MipFilter = NONE;//NONE;
AddressU = Clamp;
AddressV = Clamp;
SRGBTexture=FALSE;
MaxMipLevel=0;
MipMapLodBias=0;
};

sampler2D SamplerNoise = sampler_state
{
Texture = <texNoise>;
MinFilter = POINT;
MagFilter = POINT;
MipFilter = NONE;//NONE;
AddressU = Wrap;
AddressV = Wrap;
SRGBTexture=FALSE;
MaxMipLevel=0;
MipMapLodBias=0;
};

struct VS_OUTPUT_POST {
float4 vpos : POSITION;
float2 txcoord : TEXCOORD0;
};

struct VS_INPUT_POST {
float3 pos : POSITION;
float2 txcoord : TEXCOORD0;
};

//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
//
//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
VS_OUTPUT_POST VS_PostProcess(VS_INPUT_POST IN)
{
VS_OUTPUT_POST OUT;

float4 pos=float4(IN.pos.x,IN.pos.y,IN.pos.z,1.0);

OUT.vpos=pos;
OUT.txcoord.xy=IN.txcoord.xy;

return OUT;
}


/*============================================================================
FXAA3 QUALITY - PC
NVIDIA FXAA III.8 by TIMOTHY LOTTES
============================================================================*/

#define FXAA_LINEAR 0
#define FXAA_QUALITY__EDGE_THRESHOLD (1.0/32.0)
#define FXAA_QUALITY__EDGE_THRESHOLD_MIN (1.0/16.0)
#define FXAA_QUALITY__SUBPIX_CAP (3.0/4.0)
#define FXAA_QUALITY__SUBPIX_TRIM (7.0/8.0)
#define FXAA_QUALITY__SUBPIX_TRIM_SCALE (1.0/(1.0 - FXAA_QUALITY__SUBPIX_TRIM))
#define FXAA_SEARCH_STEPS 16
#define FXAA_SEARCH_THRESHOLD (1.0/4.0)

float4 FxaaPixelShader(VS_OUTPUT_POST IN, float2 vPos : VPOS) : COLOR
{

#define FxaaTexTop(t, p) tex2Dlod(t, float4(p, 0.0, 0.0))
#define FxaaTexOff(t, p, o, r) tex2Dlod(t, float4(p + (o * r), 0, 0))

float2 pos = IN.txcoord.xy;

float2 rcpFrame = float2(1/ScreenSize.x, ScreenSize.z/ScreenSize.x);
float4 rcpFrameOpt = float4(2/ScreenSize.x, 2*ScreenSize.z/ScreenSize.x, 0.5/ScreenSize.x, 0.5*ScreenSize.z/ScreenSize.x);

float lumaN = dot(FxaaTexOff(SamplerColor, pos.xy, float2(0, -1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1, 0), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));


float4 rgbyM;
rgbyM.xyz = FxaaTexTop(SamplerColor, pos.xy).xyz;
rgbyM.w = dot(rgbyM.xyz, float3(0.299, 0.587, 0.114));
float lumaE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1, 0), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaS = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 0, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaM = rgbyM.w;


float rangeMin = min(lumaM, min(min(lumaN, lumaW), min(lumaS, lumaE)));
float rangeMax = max(lumaM, max(max(lumaN, lumaW), max(lumaS, lumaE)));
float range = rangeMax - rangeMin;

if(range < max(FXAA_QUALITY__EDGE_THRESHOLD_MIN, rangeMax * FXAA_QUALITY__EDGE_THRESHOLD)) return rgbyM;


float lumaNW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1,-1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaNE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1,-1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaSW = dot(FxaaTexOff(SamplerColor, pos.xy, float2(-1, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));
float lumaSE = dot(FxaaTexOff(SamplerColor, pos.xy, float2( 1, 1), rcpFrame.xy).xyz, float3(0.299, 0.587, 0.114));



float lumaL = (lumaN + lumaW + lumaE + lumaS) * 0.25;
float rangeL = abs(lumaL - lumaM);
float blendL = saturate((rangeL / range) - FXAA_QUALITY__SUBPIX_TRIM) * FXAA_QUALITY__SUBPIX_TRIM_SCALE;
blendL = min(FXAA_QUALITY__SUBPIX_CAP, blendL);

float edgeVert = abs(lumaNW + (-2.0 * lumaN) + lumaNE) + 2.0 * abs(lumaW + (-2.0 * lumaM) + lumaE ) + abs(lumaSW + (-2.0 * lumaS) + lumaSE);
float edgeHorz = abs(lumaNW + (-2.0 * lumaW) + lumaSW) + 2.0 * abs(lumaN + (-2.0 * lumaM) + lumaS ) + abs(lumaNE + (-2.0 * lumaE) + lumaSE);
bool horzSpan = edgeHorz >= edgeVert;

float lengthSign = horzSpan ? -rcpFrame.y : -rcpFrame.x;
if(!horzSpan) lumaN = lumaW;
if(!horzSpan) lumaS = lumaE;
float gradientN = abs(lumaN - lumaM);
float gradientS = abs(lumaS - lumaM);
lumaN = (lumaN + lumaM) * 0.5;
lumaS = (lumaS + lumaM) * 0.5;

bool pairN = gradientN >= gradientS;
if(!pairN) lumaN = lumaS;
if(!pairN) gradientN = gradientS;
if(!pairN) lengthSign *= -1.0;
float2 posN;
posN.x = pos.x + (horzSpan ? 0.0 : lengthSign * 0.5);
posN.y = pos.y + (horzSpan ? lengthSign * 0.5 : 0.0);


gradientN *= FXAA_SEARCH_THRESHOLD;

float2 posP = posN;
float2 offNP = horzSpan ?
float2(rcpFrame.x, 0.0) :
float2(0.0f, rcpFrame.y);
float lumaEndN;
float lumaEndP;
bool doneN = false;
bool doneP = false;
posN += offNP * (-1.5);
posP += offNP * ( 1.5);
for(int i = 0; i < FXAA_SEARCH_STEPS; i++)
{
lumaEndN = dot(FxaaTexTop(SamplerColor, posN.xy).xyz, float3(0.299, 0.587, 0.114));
lumaEndP = dot(FxaaTexTop(SamplerColor, posP.xy).xyz, float3(0.299, 0.587, 0.114));
bool doneN2 = abs(lumaEndN - lumaN) >= gradientN;
bool doneP2 = abs(lumaEndP - lumaN) >= gradientN;
if(doneN2 && !doneN) posN += offNP;
if(doneP2 && !doneP) posP -= offNP;
if(doneN2 && doneP2) break;
doneN = doneN2;
doneP = doneP2;
if(!doneN) posN -= offNP * 2.0;
if(!doneP) posP += offNP * 2.0;
}

float dstN = horzSpan ? pos.x - posN.x : pos.y - posN.y;
float dstP = horzSpan ? posP.x - pos.x : posP.y - pos.y;

bool directionN = dstN < dstP;
lumaEndN = directionN ? lumaEndN : lumaEndP;

if(((lumaM - lumaN) < 0.0) == ((lumaEndN - lumaN) < 0.0))
lengthSign = 0.0;

float spanLength = (dstP + dstN);
dstN = directionN ? dstN : dstP;
float subPixelOffset = 0.5 + (dstN * (-1.0/spanLength));
subPixelOffset += blendL * (1.0/8.0);
subPixelOffset *= lengthSign;
float3 rgbF = FxaaTexTop(SamplerColor, float2(pos.x + (horzSpan ? 0.0 : subPixelOffset), pos.y + (horzSpan ? subPixelOffset : 0.0))).xyz;

#if (FXAA_LINEAR == 1)
lumaL *= lumaL;
#endif
float lumaF = dot(rgbF, float3(0.299, 0.587, 0.114)) + (1.0/(65536.0*256.0));
float lumaB = lerp(lumaF, lumaL, blendL);
float scale = min(4.0, lumaB/lumaF);
rgbF *= scale;

return float4(rgbF, lumaM);
}

float4 PS_PostProcess5(VS_OUTPUT_POST IN, float2 vPos : VPOS) : COLOR
{
float4 res;
float4 coord=0.0;

coord.xy=IN.txcoord.xy;
float4 origcolor;

coord.w=0.0;


origcolor=tex2Dlod(SamplerColor, coord);

// coord.x=IN.txcoord.x-(1.5/ScreenSize.x);
// float4 lshift=tex2Dlod(SamplerColor, coord);
// coord.x=IN.txcoord.x+(1.5/ScreenSize.x);
// float4 rshift=tex2Dlod(SamplerColor, coord);


float2 offset[8]=
{
float2(1.0, 1.0),
float2(-1.0, -1.0),
float2(-1.0, 1.0),
float2(1.0, -1.0),

float2(1.41, 0.0),
float2(-1.41, 0.0),
float2(0.0, 1.41),
float2(0.0, -1.41)
};
int i=0;

float4 tcol=origcolor;
float invscreensize=1.0/ScreenSize.x;
//for (i=0; i<8; i++) //higher quality
for (i=0; i<8; i++)
{
float2 tdir=offset.xy;
coord.xy=IN.txcoord.xy+tdir.xy*invscreensize*SamplingRange;//*1.0;
float4 ct=tex2Dlod(SamplerColor, coord);

tcol+=ct;
}
tcol*=0.111; // 1.0/(4+1)
//tcol*=0.111; // 1.0/(8+1) //higher quality


/*
//not interesting
#ifdef EBLURRING
//blur
res=tcol;
#endif
*/

//sharp
#ifdef ESHARPENING

#ifdef ESHARPENINGCOLOR
//color
res=origcolor*(1.0+((origcolor-tcol)*SharpeningAmount));
#else
//non color
float difffact=dot((origcolor.xyz-tcol.xyz), 0.333);
res=origcolor*(1.0+difffact*SharpeningAmount);
#endif

//less sharpening for bright pixels
float rgray=origcolor.z; //blue fit well
//float rgray=max(origcolor.x, max(origcolor.y, origcolor.z));
rgray=pow(rgray, 3.0);
res=lerp(res, origcolor, saturate(rgray));

#endif




//grain noise
#ifdef ENOISE
float origgray=max(res.x, res.y);//dot(res.xyz, 0.333);
origgray=max(origgray, res.z);
coord.xy=IN.txcoord.xy*16.0 + origgray;
float4 cnoi=tex2Dlod(SamplerNoise, coord);
res=lerp(res, (cnoi.x+0.5)*res, NoiseAmount*saturate(1.0-origgray*1.8));
#endif


res.w=1.0;
return res;
}

float4 PS_Process6(VS_OUTPUT_POST IN, float2 vPos : VPOS) : COLOR
{
float4 res;
float4 coord=0.0;

coord.xy=IN.txcoord.xy;
float4 origcolor;

coord.w=0.0;


origcolor=tex2Dlod(SamplerColor, coord);

// coord.x=IN.txcoord.x-(1.5/ScreenSize.x);
// float4 lshift=tex2Dlod(SamplerColor, coord);
// coord.x=IN.txcoord.x+(1.5/ScreenSize.x);
// float4 rshift=tex2Dlod(SamplerColor, coord);


float2 offset[8]=
{
float2(1.0, 1.0),
float2(-1.0, -1.0),
float2(-1.0, 1.0),
float2(1.0, -1.0),

float2(1.41, 0.0),
float2(-1.41, 0.0),
float2(0.0, 1.41),
float2(0.0, -1.41)
};
int i=0;

float4 tcol=origcolor;
float2 invscreensize=1.0/ScreenSize.x;
invscreensize.y=invscreensize.y/ScreenSize.z;
//for (i=0; i<8; i++) //higher quality
/* for (i=0; i<4; i++)
{
float2 tdir=offset.xy;
coord.xy=IN.txcoord.xy+tdir.xy*invscreensize*SamplingRange;//*1.0;
float4 ct=tex2Dlod(SamplerColor, coord);

tcol+=ct;
}
tcol*=0.2; // 1.0/(4+1)
//tcol*=0.111; // 1.0/(8+1) //higher quality
*/

coord.xy=IN.txcoord.xy;
origcolor=tex2Dlod(SamplerColor, coord);
res.y=origcolor.y;

coord.xy=IN.txcoord.xy;
coord.y-=invscreensize*ShiftSamplingRange;
origcolor=tex2Dlod(SamplerColor, coord);
res.x=origcolor.x;

coord.xy=IN.txcoord.xy;
coord.y+=invscreensize*ShiftSamplingRange;
origcolor=tex2Dlod(SamplerColor, coord);
res.z=origcolor.z;


res.w=1.0;
return res;
}

technique PostProcess
{
pass P0
{
VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 FxaaPixelShader();

FogEnable=FALSE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

technique PostProcess2
{
pass P0
{
VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 FxaaPixelShader();

FogEnable=FALSE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

technique PostProcess3
{
pass P0
{
VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 FxaaPixelShader();

FogEnable=FALSE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

technique PostProcess4
{
pass P0
{
VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 FxaaPixelShader();

FogEnable=FALSE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

technique PostProcess5
{
pass P0
{

VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 PS_PostProcess5();

DitherEnable=FALSE;
ZEnable=FALSE;
CullMode=NONE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
StencilEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

technique PostProcess6
{
pass P0
{

VertexShader = compile vs_3_0 VS_PostProcess();
PixelShader = compile ps_3_0 PS_Process6();

DitherEnable=FALSE;
ZEnable=FALSE;
CullMode=NONE;
ALPHATESTENABLE=FALSE;
SEPARATEALPHABLENDENABLE=FALSE;
AlphaBlendEnable=FALSE;
StencilEnable=FALSE;
FogEnable=FALSE;
SRGBWRITEENABLE=FALSE;
}
}

offtop:Sorry but how i can use spoiler for this forum?
p.p.s(04.05) it is good to hear that, Legacyy.

[legacyy]

Thanks that version works