Unity Real-time Reflection Related Optimization
1. The Result and Cost of the Effect after Optimization
Here is the basic background: There are 8 people on the same screen with the highest picture quality, and it takes 1.1ms (Qualcomm Snapdragon 710). Because the project is not released yet, only test figures can be used for related explanations.
2. The Original Source Code
-
Source code
using UnityEngine;
using System.Collections;
[ExecuteInEditMode]
public class MirrorReflection : MonoBehaviour
{
public Material m_matCopyDepth;
public bool m_DisablePixelLights = true;
public int m_TextureSize = 256;
public float m_ClipPlaneOffset = 0.07f;
public LayerMask m_ReflectLayers = -1;
private Hashtable m_ReflectionCameras = new Hashtable(); // Camera -> Camera table
public RenderTexture m_ReflectionTexture = null;
public RenderTexture m_ReflectionDepthTexture = null;
private int m_OldReflectionTextureSize = 0;
private static bool s_InsideRendering = false;
public void OnWillRenderObject()
{
if (!enabled || !GetComponent<Renderer>() || !GetComponent<Renderer>().sharedMaterial || !GetComponent<Renderer>().enabled)
return;
Camera cam = Camera.current;
if (!cam)
return;
// Safeguard from recursive reflections.
if (s_InsideRendering)
return;
s_InsideRendering = true;
Camera reflectionCamera;
CreateMirrorObjects(cam, out reflectionCamera);
// find out the reflection plane: position and normal in world space
Vector3 pos = transform.position;
Vector3 normal = transform.up;
// Optionally disable pixel lights for reflection
int oldPixelLightCount = QualitySettings.pixelLightCount;
if (m_DisablePixelLights)
QualitySettings.pixelLightCount = 0;
UpdateCameraModes(cam, reflectionCamera);
// Render reflection
// Reflect camera around reflection plane
float d = -Vector3.Dot(normal, pos) - m_ClipPlaneOffset;
Vector4 reflectionPlane = new Vector4(normal.x, normal.y, normal.z, d);
Matrix4x4 reflection = Matrix4x4.zero;
CalculateReflectionMatrix(ref reflection, reflectionPlane);
Vector3 oldpos = cam.transform.position;
Vector3 newpos = reflection.MultiplyPoint(oldpos);
reflectionCamera.worldToCameraMatrix = cam.worldToCameraMatrix * reflection;
// Setup oblique projection matrix so that near plane is our reflection
// plane. This way we clip everything below/above it for free.
Vector4 clipPlane = CameraSpacePlane(reflectionCamera, pos, normal, 1.0f);
Matrix4x4 projection = cam.projectionMatrix;
CalculateObliqueMatrix(ref projection, clipPlane);
reflectionCamera.projectionMatrix = projection;
reflectionCamera.cullingMask = ~(1 << 4) & m_ReflectLayers.value; // never render water layer
reflectionCamera.targetTexture = m_ReflectionTexture;
//GL.SetRevertBackfacing(true);
GL.invertCulling = true;
reflectionCamera.transform.position = newpos;
Vector3 euler = cam.transform.eulerAngles;
reflectionCamera.transform.eulerAngles = new Vector3(0, euler.y, euler.z);
reflectionCamera.depthTextureMode = DepthTextureMode.Depth;
reflectionCamera.Render();
// copy depth
Graphics.SetRenderTarget(m_ReflectionDepthTexture);
m_matCopyDepth.SetPass(0);
DrawFullscreenQuad();
Graphics.SetRenderTarget(null);
// Graphics.Blit(m_ReflectionTexture, m_ReflectionDepthTexture, m_matCopyDepth);
reflectionCamera.transform.position = oldpos;
//GL.SetRevertBackfacing(false);
GL.invertCulling = false;
Material[] materials = GetComponent<Renderer>().sharedMaterials;
foreach (Material mat in materials)
{
mat.SetTexture("_ReflectionTex", m_ReflectionTexture);
mat.SetTexture("_ReflectionDepthTex", m_ReflectionDepthTexture);
}
// // Set matrix on the shader that transforms UVs from object space into screen
// // space. We want to just project reflection texture on screen.
// Matrix4x4 scaleOffset = Matrix4x4.TRS(
// new Vector3(0.5f, 0.5f, 0.5f), Quaternion.identity, new Vector3(0.5f, 0.5f, 0.5f));
// Vector3 scale = transform.lossyScale;
// Matrix4x4 mtx = transform.localToWorldMatrix * Matrix4x4.Scale(new Vector3(1.0f / scale.x, 1.0f / scale.y, 1.0f / scale.z));
// mtx = scaleOffset * cam.projectionMatrix * cam.worldToCameraMatrix * mtx;
// foreach (Material mat in materials)
// {
// mat.SetMatrix("_ProjMatrix", mtx);
// }
// Restore pixel light count
if (m_DisablePixelLights)
QualitySettings.pixelLightCount = oldPixelLightCount;
s_InsideRendering = false;
}
// Cleanup all the objects we possibly have created
void OnDisable()
{
if (m_ReflectionTexture)
{
DestroyImmediate(m_ReflectionTexture);
m_ReflectionTexture = null;
}
if (m_ReflectionDepthTexture)
{
DestroyImmediate(m_ReflectionDepthTexture);
m_ReflectionDepthTexture = null;
}
foreach (DictionaryEntry kvp in m_ReflectionCameras)
DestroyImmediate(((Camera)kvp.Value).gameObject);
m_ReflectionCameras.Clear();
}
private void UpdateCameraModes(Camera src, Camera dest)
{
if (dest == null)
return;
// set camera to clear the same way as current camera
dest.clearFlags = src.clearFlags;
dest.backgroundColor = src.backgroundColor;
if (src.clearFlags == CameraClearFlags.Skybox)
{
Skybox sky = src.GetComponent(typeof(Skybox)) as Skybox;
Skybox mysky = dest.GetComponent(typeof(Skybox)) as Skybox;
if (!sky || !sky.material)
{
mysky.enabled = false;
}
else
{
mysky.enabled = true;
mysky.material = sky.material;
}
}
// update other values to match current camera.
// even if we are supplying custom camera&projection matrices,
// some of values are used elsewhere (e.g. skybox uses far plane)
dest.farClipPlane = src.farClipPlane;
dest.nearClipPlane = src.nearClipPlane;
dest.orthographic = src.orthographic;
dest.fieldOfView = src.fieldOfView;
dest.aspect = src.aspect;
dest.orthographicSize = src.orthographicSize;
}
// On-demand create any objects we need
private void CreateMirrorObjects(Camera currentCamera, out Camera reflectionCamera)
{
reflectionCamera = null;
// Reflection render texture
if (!m_ReflectionTexture || m_OldReflectionTextureSize != m_TextureSize)
{
if (m_ReflectionTexture)
DestroyImmediate(m_ReflectionTexture);
m_ReflectionTexture = new RenderTexture(m_TextureSize, m_TextureSize, 16);
m_ReflectionTexture.name = "__MirrorReflection" + GetInstanceID();
m_ReflectionTexture.isPowerOfTwo = true;
m_ReflectionTexture.hideFlags = HideFlags.DontSave;
m_ReflectionTexture.filterMode = FilterMode.Bilinear;
if (m_ReflectionDepthTexture)
DestroyImmediate(m_ReflectionDepthTexture);
m_ReflectionDepthTexture = new RenderTexture(m_TextureSize, m_TextureSize, 0, RenderTextureFormat.RHalf);
// m_ReflectionDepthTexture = new RenderTexture(m_TextureSize, m_TextureSize, 0, RenderTextureFormat.R8);
m_ReflectionDepthTexture.name = "__MirrorReflectionDepth" + GetInstanceID();
m_ReflectionDepthTexture.isPowerOfTwo = true;
m_ReflectionDepthTexture.hideFlags = HideFlags.DontSave;
m_ReflectionDepthTexture.filterMode = FilterMode.Bilinear;
m_OldReflectionTextureSize = m_TextureSize;
}
// Camera for reflection
reflectionCamera = m_ReflectionCameras[currentCamera] as Camera;
if (!reflectionCamera) // catch both not-in-dictionary and in-dictionary-but-deleted-GO
{
GameObject go = new GameObject("Mirror Refl Camera id" + GetInstanceID() + " for " + currentCamera.GetInstanceID(), typeof(Camera), typeof(Skybox));
reflectionCamera = go.GetComponent<Camera>();
reflectionCamera.enabled = false;
reflectionCamera.transform.position = transform.position;
reflectionCamera.transform.rotation = transform.rotation;
reflectionCamera.gameObject.AddComponent<FlareLayer>();
go.hideFlags = HideFlags.HideAndDontSave;
m_ReflectionCameras[currentCamera] = reflectionCamera;
}
}
// Extended sign: returns -1, 0 or 1 based on sign of a
private static float sgn(float a)
{
if (a > 0.0f) return 1.0f;
if (a < 0.0f) return -1.0f;
return 0.0f;
}
// Given position/normal of the plane, calculates plane in camera space.
private Vector4 CameraSpacePlane(Camera cam, Vector3 pos, Vector3 normal, float sideSign)
{
Vector3 offsetPos = pos + normal * m_ClipPlaneOffset;
Matrix4x4 m = cam.worldToCameraMatrix;
Vector3 cpos = m.MultiplyPoint(offsetPos);
Vector3 cnormal = m.MultiplyVector(normal).normalized * sideSign;
return new Vector4(cnormal.x, cnormal.y, cnormal.z, -Vector3.Dot(cpos, cnormal));
}
// Adjusts the given projection matrix so that near plane is the given clipPlane
// clipPlane is given in camera space. See article in Game Programming Gems 5 and
// http://aras-p.info/texts/obliqueortho.html
private static void CalculateObliqueMatrix(ref Matrix4x4 projection, Vector4 clipPlane)
{
Vector4 q = projection.inverse * new Vector4(
sgn(clipPlane.x),
sgn(clipPlane.y),
1.0f,
1.0f
);
Vector4 c = clipPlane * (2.0F / (Vector4.Dot(clipPlane, q)));
// third row = clip plane - fourth row
projection[2] = c.x - projection[3];
projection[6] = c.y - projection[7];
projection[10] = c.z - projection[11];
projection[14] = c.w - projection[15];
}
// Calculates reflection matrix around the given plane
private static void CalculateReflectionMatrix(ref Matrix4x4 reflectionMat, Vector4 plane)
{
reflectionMat.m00 = (1F - 2F * plane[0] * plane[0]);
reflectionMat.m01 = (-2F * plane[0] * plane[1]);
reflectionMat.m02 = (-2F * plane[0] * plane[2]);
reflectionMat.m03 = (-2F * plane[3] * plane[0]);
reflectionMat.m10 = (-2F * plane[1] * plane[0]);
reflectionMat.m11 = (1F - 2F * plane[1] * plane[1]);
reflectionMat.m12 = (-2F * plane[1] * plane[2]);
reflectionMat.m13 = (-2F * plane[3] * plane[1]);
reflectionMat.m20 = (-2F * plane[2] * plane[0]);
reflectionMat.m21 = (-2F * plane[2] * plane[1]);
reflectionMat.m22 = (1F - 2F * plane[2] * plane[2]);
reflectionMat.m23 = (-2F * plane[3] * plane[2]);
reflectionMat.m30 = 0F;
reflectionMat.m31 = 0F;
reflectionMat.m32 = 0F;
reflectionMat.m33 = 1F;
}
static public void DrawFullscreenQuad(float z = 1.0f)
{
GL.Begin(GL.QUADS);
GL.Vertex3(-1.0f, -1.0f, z);
GL.Vertex3(1.0f, -1.0f, z);
GL.Vertex3(1.0f, 1.0f, z);
GL.Vertex3(-1.0f, 1.0f, z);
GL.Vertex3(-1.0f, 1.0f, z);
GL.Vertex3(1.0f, 1.0f, z);
GL.Vertex3(1.0f, -1.0f, z);
GL.Vertex3(-1.0f, -1.0f, z);
GL.End();
}
}
Related Contents:
https://github.com/unity3d-jp/unitychan-crs/blob/master/Assets/UnityChanStage/Visualizer/MirrorReflection.cs
-
Main Function of the Script
Obtain the currently valid camera in the scene, mirror it according to the reflection plane, take pictures of the set rendering layer frame by frame, and transmit it to the reflection plane for display. Combined with the built-in translucent reflection material effect, it is superimposed on the underlying material for use.
-
Main Problem
Before optimization, it takes about 10ms or more for the reflection effect to be done. The main issues are as follows:
- The reflection effects are all real-time reflections, which are controlled by Layer. To see the reflection effect, only objects of this layer can be included in the reflection layer.
- The reflective surface is a translucent material that is independent of the real reflective surface, which consumes performance and cannot be guaranteed.
- The number of faces of characters in the project is extremely high (about 10,000 faces for a single character), and the limit is 9 characters plus pets on the same screen. In addition to its own character rendering, there are also effects such as strokes and sliced shadows. If the reflection is directly generated by taking pictures with the camera, the extra performance is very expensive.
- The rendering of the project character itself supports 5-pixel lights, with normal maps, cartoon light maps, and other complex rendering effects. If the reflection is directly generated by taking pictures with the camera, the extra performance is very expensive.
- The script itself also has some problems:
- Reflection cameras will be generated one by one according to the active cameras in the scene.
- Obtain the reflection material frame by frame and pass parameters.
- The setting of the reflection layer needs to be set once by the artist every time it is used, and it is easy to set too much or miss the setting.
3. Optimization Instructions
In response to the above problems, there are two main optimization directions for real-time reflection, art effect optimization and performance optimization.
Art Effect Optimization
1. Associate the reflection effect with the physical properties of the reflective surface.
Introduced support for reflective surface normal mappings, adding a real normal distortion effect to the reflective effect.
Method: Discard the default semi-transparent reflective surface. Directly assign the reflection map to the rendering material of the reflective surface, increase the normal effect, and use the xy channel of the normal map to offset the UV of the reflection map (the adjustment of the normal distortion strength and mask, etc. is also added here).
2. Introduce reflective surface roughness settings.
The sharpness of the reflection effect can be adjusted (blurring strategy, detailed performance optimization later).
Method: Regardless of performance, you can pass parameters to the script here, and use Gaussian blur to perform reflection blur effects on rough surfaces.
3. Introduce the reflection depth to achieve a soft transition from the present to the absence of the reflection following distance.
Method: When rendering the reflection map, pass in the attenuation coefficient, attenuate according to the distance from the vertex to the height of the ground, and write the attenuation intensity to the A channel of the reflection map.
Performance Optimization
Overview
Performance optimization mainly focuses on the following aspects:
- Reduce the amount of data and reduce the CPU burden.
- Reduce DrawCall and increase C/GPU transfer rate.
- Reduce rendering complexity and accelerate GPU rendering.
- Optimize scripts to reduce GC and repetitive operations.
Based on the above considerations, performance optimization directions are classified into two categories: art-side optimization and program-side optimization.
Art Optimization
- Dynamic and static separation
The reflection effect is divided into two parts: the static object and the baked reflection and the real-time reflection of the dynamic object. This can greatly reduce the amount of real-time computation.
- Pre-baking part
According to the reflection clarity required by the scene, pre-bake the reflection map with the smallest possible size. For very rough and unclear reflection scenes, you can share a scene reflection map.
Or the reflection maps in different states of the scene such as day and night, rain and sunny are shared. Different state effects can be achieved by adjusting material parameters such as reflection color, roughness, and reflection strength.
Fixed the scene alignment problem with the baked CubeMap through the BoxProjection method.
- Real-time part
Similar to static and baking, it can generate the smallest possible reflection map in real-time according to the reflection clarity of the scene.
(1) Program opening size adjustment:
(2) Code writing supports size configuration:
Reduce the DrawCall of real-time reflection, for example: merge character parts as much as possible, ignore smaller parts and do not reflect, etc.
Reduce the amount of data and calculation of real-time reflections, for example: use low-poly Lod to render reflections, reducing reflection effects requires reducing the number of lights, effects, etc.
- Reflective Surface Optimization Strategy
There can only be one reflective surface in the same scene. The material of the reflective surface is treated specially by the artist as a high-profile material. Expose as many parameters as possible to the artist and adapt to high-performance parameters as much as possible.
Terminal Optimization
Optimization is also made by considering several aspects of the overview.
- Remove unnecessary rendering batches and effects
In the reflection effect, because of the limited clarity, the stroke effect can be eliminated, and the shadow effect can also not be rendered.
- Reduce the computational complexity of reflection rendering
In the reflection effect, only the basic light-dark relationship needs to be shown, so only one parallel light effect is reserved. Lighting calculations are also changed from pixel lighting to vertex lighting.
A character rendering requires base color texture, normal texture, rough metallic texture, body material distinction texture, lighting texture, and buff effect texture. In reflection, the effect is reduced to only the base color texture and the buff effect texture.
To achieve the above optimization scheme, it is necessary to replace and render the material of the dynamic object when performing reflection rendering. Available strategies are as follows:
(1) Replacing the material
It is necessary to replace the rendering with the reflection material in every frame. If it is post-processing strokes, stylized monochrome, and other effects, this method is more suitable. However, there are various character materials and different parameters. Using this method, you need to maintain the reflection list of dynamic objects yourself, which is rather cumbersome.
(2) Lod Int control
It uses the Lod parameter of SubShader to control. In this way, the reflection Shader of each material can be concentrated under one Lod for calculation. The simplest way, but after testing, the consumption of real-time switching Lod is too large, which is not practical.
(3) Macro switch
It is relatively simple, with good integration and versatility, and low switching consumption. Initially, we used the optimization scheme. However, because of the multi-material and multi-pass rendering of the character, the optimization efficiency is still not enough in the later stage (the strokes and shadows cannot be optimized).
Core code:
(4) Use CommandBuff
It has the highest controllability and freedom. But the problem is the same as (1): you need to maintain the role list yourself. For occasions where role replacement occurs frequently, the maintenance cost will be higher, but it can still be achieved. When a role change occurs, the logic layer can update the role list to the reflection program for processing.
Part of the core code:
(5) Using CameraReplaceShader
The method adopted for the final optimization of our project. The advantage is that replacement rendering can be achieved, and there is no need to maintain a list of roles by yourself. The disadvantage is that the Shader of reflection rendering is rewritten.
Here is a brief explanation of the SetReplacementShader command.
He replaces all Shaders rendered in the camera according to the replacementTag set later.
For example, in the above code, we use the “MirrorReflect” Tag, so in the Shader, any SubShader containing the “MirrorReflect” Tag will be replaced with the SubShader of the same “MirrorReflect” Tag in the reflectShader.
As shown in the above example, when the camera performs reflection rendering, the SubShader containing the same “MirrorReflect” Tag can be replaced and rendered accordingly.
Program Strategic Optimization
- Combine static and real-time reflections
It is only necessary to mix dynamic and static reflections according to the stored reflection attenuation values in the A channel of the reflection map.
- Open art adjustment parameters
Some performance-related settings as mentioned above can be opened to art for selection.
- Reflection sharpness
The concept of roughness is introduced, and blur is used to cover the problem of insufficient resolution of the reflection map. The reflection resolution can be further reduced.
The blur strategy here adopts the MipMap method to avoid the consumption caused by Gaussian blur.
Program Logic Layer Performance Optimization
- Reduce the number of anti-reflection cameras
The number of reflection cameras is optimized from multi-reflection cameras generated by the camera to single-reflection cameras, and the method of frame-by-frame alignment. - Reduce Minus GC
Apply for RT, generate cameras, obtain reflection materials, transfer material parameters and other operations that can be performed at one time, and move them out of OnWillRenderObject(). - Optimize the refresh rate
Increase the refresh rate setting and change the frame-by-frame rendering to configurable refresh rate rendering. The current default is a 1/2 refresh rate. - The reflection layer terminal is written to death
To avoid art errors and speed up configuration. - Switch strategy
When the reflection plane is not visible, turn off the reflection function and release all resources (logical layer control). - Platform matching
Adapt to platforms with different performances. At present, we only enable real-time reflection effects on high-end platforms. Only the static reflection effect is enabled on the platform in the middle configuration.
Thanks to the author Luo Hanming for the contribution. Luo Hanming: Technical art expert at Nuverse Wushuang Studio.
Author’s profile page: https://www.zhihu.com/people/lucifer-3-91
YOU MAY ALSO LIKE!!!
UWA Website: https://en.uwa4d.com
UWA Blogs: https://blog.en.uwa4d.com
UWA Product: https://en.uwa4d.com/feature/got
You may also like
-
Dynamic Resolution in Games from Principle to Application
January 4, 2023 -
Introduction of the Raytracing Technology Part 2
December 21, 2022
