From 91a783dc2d63dc3f7fd0c697b54d1b309a292989 Mon Sep 17 00:00:00 2001 From: Par Winzell Date: Sun, 29 Oct 2017 22:25:14 -0700 Subject: [PATCH] Much progress with morph targets. Everything's in place to drop the progressive morph calculations into readAnimations(). Shouldn't be hard. Normals are proving trickier. I'm not convinced they make it into the FBX at all. The SDK reports the existence of normals in the primary layer, but they seems to consist exclusively of zeroes. We may have to compute them. --- src/Fbx2Raw.cpp | 219 +++++++++++++++++++++++++++---------- src/Raw2Gltf.cpp | 64 ++++++++--- src/RawModel.cpp | 2 +- src/RawModel.h | 47 +++++--- src/glTF/PrimitiveData.cpp | 20 ++-- src/glTF/PrimitiveData.h | 4 +- 6 files changed, 260 insertions(+), 96 deletions(-) diff --git a/src/Fbx2Raw.cpp b/src/Fbx2Raw.cpp index 4f9411a..d1bfa65 100644 --- a/src/Fbx2Raw.cpp +++ b/src/Fbx2Raw.cpp @@ -449,89 +449,172 @@ private: std::vector vertexJointWeights; }; +/** + * At the FBX level, each Mesh can have a set of FbxBlendShape deformers; organisational units that contain no data + * of their own. The actual deformation is determined by one or more FbxBlendShapeChannels, whose influences are all + * additively applied to the mesh. In a simpler world, each such channel would extend each base vertex with alternate + * position, and optionally normal and tangent. + * + * It's not quite so simple, though. We also have progressive morphing, where one logical morph actually consists of + * several concrete ones, each applied in sequence. For us, this means each channel contains a sequence of FbxShapes + * (aka target shape); these are the actual data-holding entities that provide the alternate vertex attributes. As such + * a channel is given more weight, it moves from one target shape to another. + * + * The total number of alternate sets of attributes, then, is the total number of target shapes across all the channels + * of all the blend shapes of the mesh. + * + * Each animation in the scene stack can yield one or zero FbxAnimCurves per channel (not target shape). We evaluate + * these curves to get the weight of the channel: this weight is further introspected on to figure out which target + * shapes we're currently interpolation between. + */ class FbxBlendShapesAccess { public: - struct Shape { - explicit Shape(const std::vector &offsets, FbxFloat defaultDeform, const std::vectoranimations) : - offsets(offsets), - defaultDeform(defaultDeform), - animations(animations) + struct BlendChannel { + explicit BlendChannel(FbxDouble defaultDeform) : + defaultDeform(defaultDeform) {} - const std::vector offsets; - const FbxDouble defaultDeform; - const std::vector animations; + struct TargetShape { + TargetShape( + double fullWeight, + const std::vector &positions, + const std::vector &normals, + const std::vector &tangents + ) : fullWeight(fullWeight), + positions(positions), + normals(normals), + tangents(tangents) + {} + + const double fullWeight; + const std::vector positions; + const std::vector normals; + const std::vector tangents; + }; + + const FbxDouble defaultDeform; + + // one for each FbxShape + std::vector targetShapes {}; + + // always the size of the scene's animation stack; can and will contain nulls + std::vector animations {}; }; explicit FbxBlendShapesAccess(const FbxScene *scene, FbxMesh *mesh) : - shapes(extractShapes(scene, mesh)) + channels(extractChannels(scene, mesh)) { } - size_t GetShapeCount() const { return shapes.size(); } - FbxDouble GetDefaultDeform(size_t shapeIx) const { - return shapes.at(shapeIx).defaultDeform; + size_t GetShapeCount() const { return channels.size(); } + FbxDouble GetDefaultDeform(size_t channelIx) const { + return channels.at(channelIx).defaultDeform; } - FbxVector4 GetPosition(size_t shapeIx, size_t vertexIx) const { - return shapes.at(shapeIx).offsets[vertexIx]; + size_t GetTargetShapeCount(size_t channelIx) const { return channels[channelIx].targetShapes.size(); } + const BlendChannel::TargetShape &GetTargetShape(size_t channelIx, size_t targetShapeIx) const { + return channels.at(channelIx).targetShapes[targetShapeIx]; } - size_t GetAnimCount(size_t shapeIx) const { return shapes.at(shapeIx).animations.size(); } - FbxAnimCurve * GetAnimation(size_t shapeIx, size_t animIx) const { - return shapes.at(shapeIx).animations[animIx]; + size_t GetAnimCount(size_t channelIx) const { return channels.at(channelIx).animations.size(); } + FbxAnimCurve *GetAnimation(size_t channelIx, size_t animIx) const { + return channels.at(channelIx).animations[animIx]; } private: - std::vector extractShapes(const FbxScene *scene, FbxMesh *mesh) const { - std::vector results; + std::vector extractChannels(const FbxScene *scene, FbxMesh *mesh) const { // acquire the regular control points from the mesh + const int controlPointsCount = mesh->GetControlPointsCount(); const FbxVector4 *meshPoints = mesh->GetControlPoints(); + + // acquire the normals, if they're present & make sure they're well-formed + FbxLayerElementArrayTemplate* meshNormals = nullptr; + if (!mesh->GetNormals(&meshNormals) || meshNormals->GetCount() != controlPointsCount) { + meshNormals = nullptr; + } + + // same, but tangents + FbxLayerElementArrayTemplate* meshTangents = nullptr; + if (!mesh->GetTangents(&meshTangents) || meshTangents->GetCount() != controlPointsCount) { + meshTangents = nullptr; + } + + std::vector results; for (int shapeIx = 0; shapeIx < mesh->GetDeformerCount(FbxDeformer::eBlendShape); shapeIx++) { - auto *blendShape = dynamic_cast(mesh->GetDeformer(shapeIx, FbxDeformer::eBlendShape)); - if (blendShape == nullptr) { + auto *fbxBlendShape = dynamic_cast(mesh->GetDeformer(shapeIx, FbxDeformer::eBlendShape)); + if (fbxBlendShape == nullptr) { continue; } - for (int channelIx = 0; channelIx < blendShape->GetBlendShapeChannelCount(); ++channelIx) { - FbxBlendShapeChannel *channel = blendShape->GetBlendShapeChannel(channelIx); - - if (channel->GetTargetShapeCount() != 1) { - if (channel->GetTargetShapeCount() > 1) { - fmt::print("Warning: Blend Shape %s is a progressive morph; this is not supported.\n", channel->GetName()); - } + for (int channelIx = 0; channelIx < fbxBlendShape->GetBlendShapeChannelCount(); ++channelIx) { + FbxBlendShapeChannel *channel = fbxBlendShape->GetBlendShapeChannel(channelIx); + unsigned int targetShapeCount = static_cast(channel->GetTargetShapeCount()); + if (targetShapeCount < 1) { continue; } - FbxShape *fbxShape = channel->GetTargetShape(0); - assert(fbxShape->GetControlPointsCount() == mesh->GetControlPointsCount()); + BlendChannel shape(channel->DeformPercent * 0.01f); - // glTF morph target positions are *mutation vectors* to be added (by weight) to the regular mesh positions. - // FBX blend shape control points, on the other hand are final positions. - // So we must do a little subtracting. - std::vector offsets; - auto shapePoints = fbxShape->GetControlPoints(); - for (int pointIx = 0; pointIx < mesh->GetControlPointsCount(); pointIx ++) { - offsets.push_back(shapePoints[pointIx] - meshPoints[pointIx]); + std::vector> targetShapes (targetShapeCount); + double *fullWeights = channel->GetTargetShapeFullWeights(); + for (int targetShapeIx = 0; targetShapeIx < targetShapes.size(); targetShapeIx++) { + FbxShape *fbxShape = channel->GetTargetShape(targetShapeIx); + assert(fbxShape->GetControlPointsCount() == controlPointsCount); + + // glTF morph target positions are *mutation vectors* to be added (by weight) to the regular mesh positions. + // FBX blend shape control points, on the other hand are final positions. + // So we must do a little subtracting. + std::vector positions, normals, tangents; + // fetch positions + const FbxVector4 *shapePoints = fbxShape->GetControlPoints(); + for (int pointIx = 0; pointIx < controlPointsCount; pointIx ++) { + positions.push_back(shapePoints[pointIx] - meshPoints[pointIx]); + } + + // maybe fetch normals + if (meshNormals) { + FbxLayerElementArrayTemplate* fbxNormals = nullptr; + if (fbxShape->GetNormals(&fbxNormals)) { + for (int pointIx = 0; pointIx < controlPointsCount; pointIx ++) { + normals.push_back(fbxNormals->GetAt(pointIx) - meshNormals->GetAt(pointIx)); + } + } + } + + // maybe fetch tangents + if (meshTangents) { + FbxLayerElementArrayTemplate* fbxTangents = nullptr; + if (fbxShape->GetTangents(&fbxTangents)) { + for (int pointIx = 0; pointIx < controlPointsCount; pointIx ++) { + tangents.push_back(fbxTangents->GetAt(pointIx) - meshTangents->GetAt(pointIx)); + } + } + } + + // finally combine all this into a TargetShape and add it to our work-in-progress BlendChannel + shape.targetShapes.push_back( + BlendChannel::TargetShape(fullWeights[targetShapeIx], positions, normals, tangents) + ); } + // go through all the animations in the scene and figure out their relevance to this mesh size_t animationCount = static_cast(scene->GetSrcObjectCount()); std::vectoranimations(animationCount); - for (int j = 0; j < animationCount; j++) { - auto *pAnimStack = scene->GetSrcObject(j); + for (int animIx = 0; animIx < animationCount; animIx++) { + auto *pAnimStack = scene->GetSrcObject(animIx); auto *layer = pAnimStack->GetMember(0); if (pAnimStack->GetMemberCount() > 1) { fmt::print("Warning: ignoring animation layers 1+ in stack %s", pAnimStack->GetName()); } - FbxAnimCurve *curve = mesh->GetShapeChannel(shapeIx, channelIx, layer, true); - // note that curve can be null here, which is fine - animations[j] = curve; + // note that some of these will be null here, which is fine; the critical part is that the + // indices maintain parity with the scene-wide animation stack + shape.animations.push_back(mesh->GetShapeChannel(shapeIx, channelIx, layer, true)); } - results.push_back(Shape(offsets, channel->DeformPercent * 0.01f, animations)); + results.push_back(shape); } } return results; } - const std::vector shapes; + const std::vector channels; }; static bool TriangleTexturePolarity(const Vec2f &uv0, const Vec2f &uv1, const Vec2f &uv2) @@ -627,9 +710,17 @@ static void ReadMesh(RawModel &raw, FbxScene *pScene, FbxNode *pNode, const std: rawSurface.jointGeometryMaxs.emplace_back(-FLT_MAX, -FLT_MAX, -FLT_MAX); } - rawSurface.defaultShapeDeforms.clear(); - for (int shapeIx = 0; shapeIx < blendShapes.GetShapeCount(); shapeIx ++) { - rawSurface.defaultShapeDeforms.push_back(blendShapes.GetDefaultDeform(shapeIx)); + rawSurface.blendChannels.clear(); + for (size_t shapeIx = 0; shapeIx < blendShapes.GetShapeCount(); shapeIx ++) { + for (size_t targetIx = 0; targetIx < blendShapes.GetTargetShapeCount(shapeIx); targetIx ++) { + const auto &shape = blendShapes.GetTargetShape(shapeIx, targetIx); + float defaultDeform = static_cast(blendShapes.GetDefaultDeform(shapeIx)); + rawSurface.blendChannels.push_back(RawBlendChannel { + defaultDeform, + !shape.normals.empty(), + !shape.tangents.empty() + }); + } } int polygonVertexIndex = 0; @@ -732,10 +823,21 @@ static void ReadMesh(RawModel &raw, FbxScene *pScene, FbxNode *pNode, const std: rawSurface.bounds.AddPoint(vertex.position); - for (int shapeIx = 0; shapeIx < blendShapes.GetShapeCount(); shapeIx ++) { - // extract this vertex' position in this blend shape, transform it as per above, and add it to the vector - const Vec3f rawPos = toVec3(transform.MultNormalize(blendShapes.GetPosition(shapeIx, controlPointIndex))); - vertex.shapePositions.push_back(rawPos); + for (size_t shapeIx = 0; shapeIx < blendShapes.GetShapeCount(); shapeIx ++) { + for (size_t targetIx = 0; targetIx < blendShapes.GetTargetShapeCount(shapeIx); targetIx ++) { + const auto &shape = blendShapes.GetTargetShape(shapeIx, targetIx); + + RawBlendVertex blendVertex; + // the morph target positions must be transformed just as with the vertex positions above + blendVertex.position = toVec3(transform.MultNormalize(shape.positions[controlPointIndex])); + if (!shape.normals.empty()) { + blendVertex.normal = toVec3(shape.normals[controlPointIndex]); + } + if (!shape.tangents.empty()) { + blendVertex.tangent = toVec4(shape.tangents[controlPointIndex]); + } + vertex.blends.push_back(blendVertex); + } } if (skinning.IsSkinned()) { @@ -943,12 +1045,12 @@ static void ReadAnimations(RawModel &raw, FbxScene *pScene) { FbxTime::EMode eMode = FbxTime::eFrames24; const int animationCount = pScene->GetSrcObjectCount(); - for (int animIx = 0; animIx < animationCount; animIx++) { + for (size_t animIx = 0; animIx < animationCount; animIx++) { FbxAnimStack *pAnimStack = pScene->GetSrcObject(animIx); FbxString animStackName = pAnimStack->GetName(); if (verboseOutput) { - fmt::printf("animation %d: %s (%d%%)", animIx, (const char *) animStackName, 0); + fmt::printf("animation %zu: %s (%d%%)", animIx, (const char *) animStackName, 0); } pScene->SetCurrentAnimationStack(pAnimStack); @@ -991,8 +1093,8 @@ static void ReadAnimations(RawModel &raw, FbxScene *pScene) if (nodeAttr != nullptr && nodeAttr->GetAttributeType() == FbxNodeAttribute::EType::eMesh) { // it's inelegant to recreate this same access class multiple times, but it's also dirt cheap... FbxBlendShapesAccess blendShapes(pScene, dynamic_cast(nodeAttr)); - for (size_t shapeIx = 0; shapeIx < blendShapes.GetShapeCount(); shapeIx ++) { - shapeAnimCurves.push_back(blendShapes.GetAnimation(shapeIx, animIx)); + for (size_t channelIx = 0; channelIx < blendShapes.GetShapeCount(); channelIx ++) { + shapeAnimCurves.push_back(blendShapes.GetAnimation(channelIx, animIx)); } } for (FbxLongLong frameIndex = firstFrameIndex; frameIndex <= lastFrameIndex; frameIndex++) { @@ -1024,8 +1126,11 @@ static void ReadAnimations(RawModel &raw, FbxScene *pScene) channel.scales.push_back(toVec3f(localScale)); for (auto curve : shapeAnimCurves) { - float weight = curve->Evaluate(pTime) * 0.01; // [0, 100] in FBX, [0, 1] in glTF - hasMorphs |= (fabs(weight) > epsilon); + float weight = 0f; + if (curve) { + weight = curve->Evaluate(pTime) * 0.01f; // [0, 100] in FBX, [0, 1] in glTF + hasMorphs |= (fabs(weight) > epsilon); + } channel.weights.push_back(weight); } } diff --git a/src/Raw2Gltf.cpp b/src/Raw2Gltf.cpp index 5720fcf..cc479cc 100644 --- a/src/Raw2Gltf.cpp +++ b/src/Raw2Gltf.cpp @@ -499,7 +499,11 @@ ModelData *Raw2Gltf( mesh = meshIter->second.get(); } else { - auto meshPtr = gltf->meshes.hold(new MeshData(rawSurface.name, rawSurface.defaultShapeDeforms)); + std::vector defaultDeforms; + for (const auto &channel : rawSurface.blendChannels) { + defaultDeforms.push_back(channel.defaultDeform); + } + auto meshPtr = gltf->meshes.hold(new MeshData(rawSurface.name, defaultDeforms)); meshByNodeName[nodeName] = meshPtr; meshNode.SetMesh(meshPtr->ix); mesh = meshPtr.get(); @@ -568,21 +572,6 @@ ModelData *Raw2Gltf( accessor->min = toStdVec(rawSurface.bounds.min); accessor->max = toStdVec(rawSurface.bounds.max); - - for (int ii = 0; ii < rawSurface.defaultShapeDeforms.size(); ii ++) { - auto bufferView = gltf->GetAlignedBufferView(buffer, BufferViewData::GL_ARRAY_BUFFER); - std::vector result; - Bounds shapeBounds; - for (int jj = 0; jj < surfaceModel.GetVertexCount(); jj ++) { - const Vec3f &position = surfaceModel.GetVertex(jj).shapePositions[ii]; - shapeBounds.AddPoint(position); - result.push_back(position); - } - accessor = gltf->AddAccessorWithView(*bufferView, GLT_VEC3F, result); - accessor->min = toStdVec(shapeBounds.min); - accessor->max = toStdVec(shapeBounds.max); - primitive->AddTarget(*accessor); - } } if ((surfaceModel.GetVertexAttributes() & RAW_VERTEX_ATTRIBUTE_NORMAL) != 0) { const AttributeDefinition ATTR_NORMAL("NORMAL", &RawVertex::normal, @@ -618,6 +607,49 @@ ModelData *Raw2Gltf( GLT_VEC4F, draco::GeometryAttribute::GENERIC, draco::DT_FLOAT32); gltf->AddAttributeToPrimitive(buffer, surfaceModel, *primitive, ATTR_WEIGHTS); } + + // each channel present in the mesh always ends up a target in the primitive + for (int channelIx = 0; channelIx < rawSurface.blendChannels.size(); channelIx ++) { + const auto &channel = rawSurface.blendChannels[channelIx]; + + // track the bounds of each shape channel + Bounds shapeBounds; + + std::vector positions, normals; + std::vector tangents; + for (int jj = 0; jj < surfaceModel.GetVertexCount(); jj ++) { + auto blendVertex = surfaceModel.GetVertex(jj).blends[channelIx]; + shapeBounds.AddPoint(blendVertex.position); + positions.push_back(blendVertex.position); + if (channel.hasNormals) { + normals.push_back(blendVertex.normal); + } + if (channel.hasTangents) { + tangents.push_back(blendVertex.tangent); + } + } + std::shared_ptr pAcc = gltf->AddAccessorWithView( + *gltf->GetAlignedBufferView(buffer, BufferViewData::GL_ARRAY_BUFFER), + GLT_VEC3F, positions); + pAcc->min = toStdVec(shapeBounds.min); + pAcc->max = toStdVec(shapeBounds.max); + + std::shared_ptr nAcc; + if (channel.hasNormals) { + nAcc = gltf->AddAccessorWithView( + *gltf->GetAlignedBufferView(buffer, BufferViewData::GL_ARRAY_BUFFER), + GLT_VEC3F, normals); + } + + std::shared_ptr tAcc; + if (channel.hasTangents) { + nAcc = gltf->AddAccessorWithView( + *gltf->GetAlignedBufferView(buffer, BufferViewData::GL_ARRAY_BUFFER), + GLT_VEC4F, tangents); + } + + primitive->AddTarget(pAcc.get(), nAcc.get(), tAcc.get()); + } } if (options.useDraco) { // Set up the encoder. diff --git a/src/RawModel.cpp b/src/RawModel.cpp index 44bba64..d9e2e9f 100644 --- a/src/RawModel.cpp +++ b/src/RawModel.cpp @@ -38,7 +38,7 @@ bool RawVertex::operator==(const RawVertex &other) const (jointIndices == other.jointIndices) && (jointWeights == other.jointWeights) && (polarityUv0 == other.polarityUv0) && - (shapePositions == other.shapePositions); + (blends == other.blends); } size_t RawVertex::Difference(const RawVertex &other) const diff --git a/src/RawModel.h b/src/RawModel.h index 3f0dc78..a1a2b83 100644 --- a/src/RawModel.h +++ b/src/RawModel.h @@ -28,6 +28,19 @@ enum RawVertexAttribute RAW_VERTEX_ATTRIBUTE_AUTO = 1 << 31 }; +struct RawBlendVertex +{ + Vec3f position {}; + Vec3f normal {}; + Vec4f tangent {}; + + bool operator==(const RawBlendVertex &other) const { + return position == other.position && + normal == other.normal && + tangent == other.tangent; + } +}; + struct RawVertex { RawVertex() : @@ -46,7 +59,9 @@ struct RawVertex Vec4i jointIndices { 0, 0, 0, 0 }; Vec4f jointWeights { 0.0f }; - std::vector shapePositions { }; + // each vertex can have many alternate positions, normals and tangents -- one set per blend shape target. + // the size of this vector is always identical to the size of RawSurface.blendChannels + std::vector blends { }; bool polarityUv0; bool pad1; @@ -158,24 +173,30 @@ struct RawMaterial int textures[RAW_TEXTURE_USAGE_MAX]; }; +struct RawBlendChannel +{ + float defaultDeform; + bool hasNormals; + bool hasTangents; +}; + struct RawSurface { - std::string name; // The name of this surface - std::string nodeName; // The node that links to this surface. - std::string skeletonRootName; // The name of the root of the skeleton. - Bounds bounds; - std::vector jointNames; - std::vector jointGeometryMins; - std::vector jointGeometryMaxs; - std::vector inverseBindMatrices; - std::vector defaultShapeDeforms; - bool discrete; - bool skinRigid; + std::string name; // The name of this surface + std::string nodeName; // The node that links to this surface. + std::string skeletonRootName; // The name of the root of the skeleton. + Bounds bounds; + std::vector jointNames; + std::vector jointGeometryMins; + std::vector jointGeometryMaxs; + std::vector inverseBindMatrices; + std::vector blendChannels; + bool discrete; }; struct RawChannel { - int nodeIndex; + int nodeIndex; std::vector translations; std::vector rotations; std::vector scales; diff --git a/src/glTF/PrimitiveData.cpp b/src/glTF/PrimitiveData.cpp index ef91426..9423117 100644 --- a/src/glTF/PrimitiveData.cpp +++ b/src/glTF/PrimitiveData.cpp @@ -39,9 +39,13 @@ void PrimitiveData::NoteDracoBuffer(const BufferViewData &data) dracoBufferView = data.ix; } -void PrimitiveData::AddTarget(const AccessorData &positions) +void PrimitiveData::AddTarget(const AccessorData *positions, const AccessorData *normals, const AccessorData *tangents) { - targetPositionAccessors.push_back(positions.ix); + targetAccessors.push_back({ + positions->ix, + normals ? normals->ix : -1, + tangents ? tangents ->ix : -1 + }); } void to_json(json &j, const PrimitiveData &d) { @@ -53,12 +57,14 @@ void to_json(json &j, const PrimitiveData &d) { if (d.indices >= 0) { j["indices"] = d.indices; } - if (!d.targetPositionAccessors.empty()) { + if (!d.targetAccessors.empty()) { json targets {}; - for (int ii = 0; ii < d.targetPositionAccessors.size(); ii ++) { - targets.push_back({ - { "POSITION", d.targetPositionAccessors[ii] } - }); + int pIx, nIx, tIx; + for (auto accessor : d.targetAccessors) { + std::tie(pIx, nIx, tIx) = accessor; + if (pIx >= 0) { targets.push_back({{ "POSITION", pIx }}); } + if (nIx >= 0) { targets.push_back({{ "NORMAL", nIx }}); } + if (tIx >= 0) { targets.push_back({{ "TANGENT", tIx }}); } } j["targets"] = targets; } diff --git a/src/glTF/PrimitiveData.h b/src/glTF/PrimitiveData.h index 64053c2..892c662 100644 --- a/src/glTF/PrimitiveData.h +++ b/src/glTF/PrimitiveData.h @@ -31,7 +31,7 @@ struct PrimitiveData void AddAttrib(std::string name, const AccessorData &accessor); - void AddTarget(const AccessorData &positions); + void AddTarget(const AccessorData *positions, const AccessorData *normals, const AccessorData *tangents); template void AddDracoAttrib(const AttributeDefinition attribute, const std::vector &attribArr) @@ -61,7 +61,7 @@ struct PrimitiveData const unsigned int material; const MeshMode mode; - std::vector targetPositionAccessors { }; + std::vector> targetAccessors {}; std::map attributes; std::map dracoAttributes;