Action Units

Action Units#

To use the Action Units feature the input .fbx file should have blend shape targets with the names corresponding to the supported Action Units:

Name	Support Status
BrowDownRight	OK
BrowDownLeft	OK
BrowInnerUp	OK
BrowOuterUpRight	OK
BrowOuterUpLeft	OK
CheekPuff	-
CheekSquintRight	OK
CheekSquintLeft	OK
JawForward	-
JawRight	OK
JawLeft	OK
JawOpen	OK
MouthClose	-
MouthFunnel	-
MouthPucker	-
MouthRight	OK
MouthLeft	OK
MouthSmileRight	OK
MouthSmileLeft	OK
MouthDimpleRight	-
MouthDimpleLeft	-
MouthRollUpper	-
MouthShrugUpper	-
MouthShrugLower	OK
MouthRollLower	-
MouthFrownRight	-
MouthFrownLeft	-
MouthUpperUpRight	-
MouthUpperUpLeft	-
MouthLowerDownRight	-
MouthLowerDownLeft	-
NoseSneerRight	OK
NoseSneerLeft	OK
MouthPressRight	-
MouthPressLeft	-
MouthStretchRight	OK
MouthStretchLeft	OK
EyeBlinkRight	-
EyeBlinkLeft	-
EyeWideRight	-
EyeWideLeft	-
EyeSquintRight	OK
EyeSquintLeft	OK
EyeLookDownRight	-
EyeLookInRight	-
EyeLookOutRight	-
EyeLookUpRight	-
EyeLookDownLeft	-
EyeLookInLeft	-
EyeLookOutLeft	-
EyeLookUpLeft	-

For more details about these targets refer to the ARKit documentation

note

It's not necessary to provide the blend shapes for all supported Action Units.

Effect build#

Create your Action Unit effect following the steps described in the section Animated 3d mask.
Convert the .fbx file with action units blend shapes manually into .bsm2 mesh and .ktx texture with blend shapes data:
fbx_bsm2 groot.FBX
The command above produces groot.bsm2 and bs_groot_nomip_point.ktx files and writes to console the list of the blend shape names:
Blend shapes (41):
0 BrowDownRight
1 BrowDownLeft
2 BrowInnerUp
3 BrowOuterUpRight
4 BrowOuterUpLeft
5 CheekPuff
6 CheekSquintRight
7 CheekSquintLeft
8 JawForward
9 JawRight
10 JawLeft
11 JawOpen
12 MouthClose
13 MouthFunnel
14 MouthPucker
15 MouthRight
16 MouthLeft
17 MouthSmileRight
18 MouthSmileLeft
19 MouthRollUpper
20 MouthRollLower
21 MouthShrugUpper
22 MouthShrugLower
23 MouthFrownRight
24 MouthFrownLeft
25 MouthUpperUpRight
26 MouthUpperUpLeft
27 MouthLowerDownRight
28 MouthLowerDownLeft
29 NoseSneerRight
30 NoseSneerLeft
31 MouthPressRight
32 MouthPressLeft
33 MouthStretchRight
34 MouthStretchLeft
35 EyeBlinkRight
36 EyeBlinkLeft
37 EyeWideRight
38 EyeWideLeft
39 EyeSquintRight
40 EyeSquintLeft
Filter Editor supports models with action units and puts groot.bsm2 and bs_groot_nomip_point.ktx files to the effect folder after converting.
Open effect config.json and add the action units array with its names listed in the same order as in the console output of fbx_bsm2, and add recognizer: ["action_units"] property.
"recognizer": [
"action_units"
],
"action_units" : [
"BrowDownRight",
"BrowDownLeft",
"BrowInnerUp",
"BrowOuterUpRight",
"BrowOuterUpLeft",
"CheekPuff",
"CheekSquintRight",
"CheekSquintLeft",
"JawForward",
"JawRight",
"JawLeft",
"JawOpen",
"MouthClose",
"MouthFunnel",
"MouthPucker",
"MouthRight",
"MouthLeft",
"MouthSmileRight",
"MouthSmileLeft",
"MouthRollUpper",
"MouthRollLower",
"MouthShrugUpper",
"MouthShrugLower",
"MouthFrownRight",
"MouthFrownLeft",
"MouthUpperUpRight",
"MouthUpperUpLeft",
"MouthLowerDownRight",
"MouthLowerDownLeft",
"NoseSneerRight",
"NoseSneerLeft",
"MouthPressRight",
"MouthPressLeft",
"MouthStretchRight",
"MouthStretchLeft",
"EyeBlinkRight",
"EyeBlinkLeft",
"EyeWideRight",
"EyeWideLeft",
"EyeSquintRight",
"EyeSquintLeft"]
Filter Editor generates action_units property if model contains action units data.
Copy the created bs_groot_nomip_point.ktx file to the effect folder and assign it to tex_blend_shapes sampler for all materials from this .fbx in cfg.toml.
[materials.eye]
vs = "blend_shape.vert"
fs = "pbr.frag"
blend = "off"
backfaces = false
colorwrite = true
zwrite = true
shadow = false
[materials.eye.samplers]
tex_diffuse="eye.png"
tex_normal="blinn2_Normal.png"
tex_metallic="blinn2_Metallic.png"
tex_roughness="blinn2_Roughness.png"
tex_blend_shapes = "bs_groot_nomip_point.ktx" # there is bs_groot_nomip_point.ktx should be assigned.
note
tex_blend_shapes is the name of 2DArray samlper in vertex shader. If you use custom shader make sure that the name of 2DArray samlper in shader and the name of the sampler in cfg.toml are the same.
Change the vertex shaders of all materials from this .fbx to:
#version 300 es

precision mediump sampler2DArray;

#define GLFX_TBN
#define GLFX_LIGHTING

layout( location = 0 ) in vec3 attrib_pos;
#ifdef GLFX_LIGHTING
layout( location = 1 ) in vec3 attrib_n;
#ifdef GLFX_TBN
layout( location = 2 ) in vec4 attrib_t;
#endif
#endif
layout( location = 3 ) in vec2 attrib_uv;
layout( location = 4 ) in uvec4 attrib_bones;
#ifndef GLFX_1_BONE
layout( location = 5 ) in vec4 attrib_weights;
#endif
#ifdef GLFX_MALI_VERTEX_ID_ATTRIB
layout( location = 6 ) in uint attrib_vertex_id;
#endif

layout(std140) uniform glfx_GLOBAL
{
mat4 glfx_MVP;
mat4 glfx_PROJ;
mat4 glfx_MV;
vec4 glfx_VIEW_Q;
};
layout(std140) uniform glfx_INSTANCES
{
vec4 glfx_IDATA[48];
};
uniform uint glfx_CURRENT_I;
#define glfx_T_SPAWN (glfx_IDATA[glfx_CURRENT_I].x)
#define glfx_T_ANIM (glfx_IDATA[glfx_CURRENT_I].y)
#define glfx_ANIMKEY (glfx_IDATA[glfx_CURRENT_I].z)
#define glfx_IDX_OFFSET (glfx_IDATA[glfx_CURRENT_I].w)
layout(std140) uniform glfx_ACTION_UNITS
{
vec4 glfx_AU[13];
};

uniform sampler2D glfx_BONES;

uniform sampler2DArray tex_blend_shapes;

out vec2 var_uv;
#ifdef GLFX_LIGHTING
#ifdef GLFX_TBN
out vec3 var_t;
out vec3 var_b;
#endif
out vec3 var_n;
out vec3 var_v;
#endif

mat3x4 get_bone( uint bone_idx, int y )
{
int b = int(bone_idx)*3;
mat3x4 m = mat3x4(
texelFetch( glfx_BONES, ivec2(b,y), 0 ),
texelFetch( glfx_BONES, ivec2(b+1,y), 0 ),
texelFetch( glfx_BONES, ivec2(b+2,y), 0 ) );
return m;
}

mat3x4 get_transform()
{
int y = int(glfx_ANIMKEY);
mat3x4 m = get_bone( attrib_bones[0], y );
#ifndef GLFX_1_BONE
if( attrib_weights[1] > 0. )
{
m = m*attrib_weights[0] + get_bone( attrib_bones[1], y )*attrib_weights[1];
if( attrib_weights[2] > 0. )
{
m += get_bone( attrib_bones[2], y )*attrib_weights[2];
if( attrib_weights[3] > 0. )
m += get_bone( attrib_bones[3], y )*attrib_weights[3];
}
}
#endif
return m;
}

mat3 shortest_arc_m3( vec3 from, vec3 to )
{
vec3 a = cross( from, to );
float c = dot( from, to );

float t = 1./(1.+c);
float tx = t*a.x;
float ty = t*a.y;
float tz = t*a.z;
float txy = tx*a.y;
float txz = tx*a.z;
float tyz = ty*a.z;

return mat3
(
c + tx*a.x, txy + a.z, txz - a.y,
txy - a.z, c + ty*a.y, tyz + a.x,
txz + a.y, tyz - a.x, c + tz*a.z
);
}

void main()
{
vec3 vpos = attrib_pos;
#ifdef GLFX_MALI_VERTEX_ID_ATTRIB
int vertex_idx = int(attrib_vertex_id) - int(glfx_IDX_OFFSET);
#else
int vertex_idx = gl_VertexID - int(glfx_IDX_OFFSET);
#endif
ivec2 bs_p_uv = ivec2((vertex_idx&31)<<1,vertex_idx>>5);
#ifdef GLFX_LIGHTING
vec3 vn = attrib_n;
ivec2 bs_n_uv = ivec2(bs_p_uv.x+1,bs_p_uv.y);
#endif

int au_size = textureSize( tex_blend_shapes, 0 ).z;
for( int i = 0; i != au_size; ++i )
{
float bs_w = glfx_AU[i>>2][i&3];
if( bs_w != 0. )
{
vec3 bs_p_delta = texelFetch( tex_blend_shapes, ivec3(bs_p_uv,i), 0 ).xyz*bs_w;
vpos += bs_p_delta;
#ifdef GLFX_LIGHTING
vec3 bs_n_delta = texelFetch( tex_blend_shapes, ivec3(bs_n_uv,i), 0 ).xyz*bs_w;
vn += bs_n_delta;
#endif
}
}

mat3x4 m = get_transform();
vpos = vec4(vpos,1.)*m;

gl_Position = glfx_MVP * vec4(vpos,1.);

var_uv = attrib_uv;

#ifdef GLFX_LIGHTING
vn = normalize(vn);
var_n = mat3(glfx_MV)*(vn*mat3(m));
#ifdef GLFX_TBN
vec3 vt = shortest_arc_m3(attrib_n,vn)*attrib_t.xyz;
var_t = mat3(glfx_MV)*(vt*mat3(m));
var_b = attrib_t.w*cross( var_n, var_t );
#endif
var_v = (glfx_MV*vec4(vpos,1.)).xyz;
#endif
}