shaders - outline object effect

How can I achieve an outline effect similar to the ones found in League of Legends or Diablo III?

Is it done using a shader? How?
I would prefer answers that are not tied up to any particular engine but one that I can adapt to whatever engine I'm working on.

Answer

You will have to render the object twice at some point. You can get away with rendering just the faces facing the camera once and the faces facing away from the camera once, but it has its trade-offs.

The most simplest common solution is done by rendering the object twice in the same pass:

• You use a vertex shader to invert the normals of the object and "blow it up" by the size of the outline and a fragment shader to render it in the outline colour


• Over that outline render, you render the object normally. The z-order is usually automatically right, more or less, since the outline is made by the faces which are in the "back" of the object while the figure itself is made up of faces facing the camera.

This is simple enough to build and implement and avoids any render-to-texture tricks, but has a few noticeable drawbacks:

• The outline size, if you don't scale it by the distance from the camera, will vary. Objects further away will have a smaller outline than those nearby. Of course, this might be what you actually want.


• The "blow up" vertex shader doesn't work very well for complex objects like the skeleton in you example, easily introducing z-fighting artefacts to the render. Fixing it requires you to render the object in two passes, but saves you from reversing the normals.


• The outline and the object might not work very well when other objects are occupying the same space and are in general a pain to get right when combined with reflection and refraction shaders.

The basic idea for such a shader looks like this (Cg, for Unity - the code is a slightly modified toon shader I found somewhere and didn't note the source, so it's more badly-written proof-of-concept than a ready-to-use shader):

Shader "Basic Outline" {
    Properties {

        _Color ("Main Color", Color) = (.5,.5,.5,1)
        _OutlineColor ("Outline Color", Color) = (1,0.5,0,1)
        _Outline ("Outline width", Range (0.0, 0.1)) = .05
        _MainTex ("Base (RGB)", 2D) = "white" { }
    }
    SubShader {
        Tags { "RenderType"="Opaque" }
        Pass {
            Name "OUTLINE"
            Tags { "LightMode" = "Always" }

CGPROGRAM
#pragma exclude_renderers gles
#pragma exclude_renderers xbox360
#pragma vertex vert

struct appdata {
    float4 vertex;
    float3 normal;
};


struct v2f
{
    float4 pos : POSITION;
    float4 color : COLOR;
    float fog : FOGC;
};
float _Outline;
float4 _OutlineColor;
v2f vert(appdata v)
{

    v2f o;
    o.pos = mul(UNITY_MATRIX_MVP, v.vertex);
    float3 norm = mul ((float3x3)UNITY_MATRIX_MV, v.normal);
    norm.x *= UNITY_MATRIX_P[0][0];
    norm.y *= UNITY_MATRIX_P[1][1];
    o.pos.xy += norm.xy * _Outline;
    o.fog = o.pos.z;
    o.color = _OutlineColor;
    return o;
}

ENDCG
            Cull Front
            ZWrite On
            ColorMask RGB
            Blend SrcAlpha OneMinusSrcAlpha
            SetTexture [_MainTex] { combine primary }
        }
        Pass {
        Name "BASE"
        Tags {"LightMode" = "Always"}

CGPROGRAM
#pragma fragment frag
#pragma vertex vert
#pragma fragmentoption ARB_fog_exp2
#pragma fragmentoption ARB_precision_hint_fastest
#include "UnityCG.cginc"

struct v2f {
    float4 pos : SV_POSITION;
    float2    uv            : TEXCOORD0;

    float3    viewDir        : TEXCOORD1;
    float3    normal        : TEXCOORD2;
}; 

v2f vert (appdata_base v)
{
    v2f o;
    o.pos = mul (UNITY_MATRIX_MVP, v.vertex);
    o.normal = v.normal;
    o.uv = TRANSFORM_UV(0);

    o.viewDir = ObjSpaceViewDir( v.vertex );
    return o;
}

uniform float4 _Color;

uniform sampler2D _MainTex;
float4 frag (v2f i)  : COLOR
{
    half4 texcol = tex2D( _MainTex, i.uv );


    half3 ambient = texcol.rgb * (UNITY_LIGHTMODEL_AMBIENT.rgb);
    return float4( ambient, texcol.a * _Color.a );
}
ENDCG
    }
    }
    FallBack "Diffuse"
}

The other common method renders the object twice as well, but avoids the vertex shader fully. On the other hand, it can't be easily done in a single pass, and needs render-to-texture: Render the object once with a "flat" outline-coloured fragment shader and use a (weighted) blur on this render in screen space, then render the object as usual on top of it.

There's also a third and possibly easiest to implement method, though it taxes the GPU a bit more and will make your artists want to murder you in your sleep unless you make it easy for them to generate: Have the objects have the outline as a separate mesh all the time, just fully transparent or moved somewhere where it's not seen (like deep underground) until you need it