BNBEffectPlayer

Objective-C

@interface BNBEffectPlayer : NSObject

+ (nullable BNBEffectPlayer *)create:(nullable BNBEffectPlayerConfiguration *)configuration;

/**
 * Add callback to receive FPS information.
 * Thread-safe. May be called from any thread
 */
- (void)addFrameDurationListener:(nullable id<BNBFrameDurationListener>)frameDurationListener;

/**
 * Remove callback to receive FPS information.
 * Thread-safe. May be called from any thread
 */
- (void)removeFrameDurationListener:(nullable id<BNBFrameDurationListener>)frameDurationListener;

/**
 * Add callback to receive faces count in frame.
 * Thread-safe. May be called from any thread
 */
- (void)addFaceNumberListener:(nullable id<BNBFaceNumberListener>)faceNumberListener;

/**
 * Remove callback to receive faces count in frame.
 * Thread-safe. May be called from any thread
 */
- (void)removeFaceNumberListener:(nullable id<BNBFaceNumberListener>)faceNumberListener;

/**
 * Add callback to receive frame data right after processing in recognizer.
 * Thread-safe. May be called from any thread
 */
- (void)addFrameDataListener:(nullable id<BNBFrameDataListener>)frameDataListener;

/**
 * Remove callback to receive frame data right after processing in recognizer.
 * Thread-safe. May be called from any thread
 */
- (void)removeFrameDataListener:(nullable id<BNBFrameDataListener>)frameDataListener;

/**
 * Add callback to receive center of the face in frame relative to top-left
 * corner in [0, 1] space.
 * Thread-safe. May be called from any thread
 */
- (void)addCameraPoiListener:(nullable id<BNBCameraPoiListener>)cameraPoiListener;

/**
 * Remove callback to receive center of the face in frame relative to top-left
 * Thread-safe. May be called from any thread
 */
- (void)removeCameraPoiListener:(nullable id<BNBCameraPoiListener>)cameraPoiListener;

/**
 * Add callback to receive current effect info from Effect Player.
 * Thread-safe. May be called from any thread
 */
- (void)addEffectInfoListener:(nullable id<BNBEffectInfoListener>)effectInfoListener;

/**
 * Remove callback to receive current effect info from Effect Player.
 * Thread-safe. May be called from any thread
 */
- (void)removeEffectInfoListener:(nullable id<BNBEffectInfoListener>)effectInfoListener;

/**
 * Add callback to receive current effect activation notification from Effect Player.
 * Thread-safe. May be called from any thread
 */
- (void)addEffectActivationCompletionListener:(nullable id<BNBEffectActivationCompletionListener>)effectActivationCompletionListener;

/**
 * Remove callback to receive current effect activation notification from Effect Player.
 * Thread-safe. May be called from any thread
 */
- (void)removeEffectActivationCompletionListener:(nullable id<BNBEffectActivationCompletionListener>)effectActivationCompletionListener;

/**
 * Sets maximum allowed face results, if face tracking feature is present.
 * Thread-safe. May be called from any thread
 * This option is relevant only for GLFX effects.
 */
- (void)setMaxFaces:(int32_t)maxFaces;

/**
 * Set OpenGL name of input external texture.
 * https://developer.android.com/reference/android/graphics/SurfaceTexture .
 * MUST be called from the render thread
 *
 * @param name Texture name
 * @param width Textute width
 * @param height Texture height
 *
 * @see setUseExtCamTex
 *
 * <br/> Note: This function is Android only.
 * <br/> Note: We use texture size to calculate aspect ratio only.
 */
- (void)setExternalCameraTexture:(int32_t)name
                           width:(int32_t)width
                          height:(int32_t)height;

/**
 * Use external texture as input source.
 * MUST be called from the render thread
 * @see setExternalCameraTexture
 * @note This function is Android only
 */
- (void)setUseExtCamTex:(BOOL)value;

/**
 * Use to notify the EffectPlayer that the surface exists and effect can be played.
 * 0, 0 for width and height are valid values.
 * MUST be called from the render thread
 */
- (void)surfaceCreated:(int32_t)width
                height:(int32_t)height;

/**
 * Notify about rendering surface being resized.
 * MUST be called from the render thread
 */
- (void)surfaceChanged:(int32_t)width
                height:(int32_t)height;

/**
 * This method should be called right before an active context will become invalid.
 * Switches playback state to inactive state. If it's not done an application will be
 * crashed on next draw iteration. After losing the surface effect playback can't be
 * resumed from last position.
 * MUST be called from the render thread
 */
- (void)surfaceDestroyed;

/**
 * Draw the current effect into the current OpenGL framebuffer. Uses internal frame_data
 * object obtained from latest push_frame recognition result.
 *
 * Return current frame number if drawing was performed and caller should swap buffers
 * otherwise `DRAW_SKIPPED`(-1)
 * MUST be called from the render thread
 */
- (int64_t)draw;

/**
 * Draw the current effect into the current OpenGL framebuffer. Uses externally provided frameData
 * object instead of internal one obtained from latest push_frame recognition result.
 *
 * Return frame number from provided frameData if drawing was performed and caller should swap buffers
 * otherwise `DRAW_SKIPPED`(-1)
 * MUST be called from the render thread
 */
- (int64_t)drawWithExternalFrameData:(nullable BNBFrameData *)frameData;

/**
 * Record last rendering ("draw_()") result into current OpenGL framebuffer.
 * Content is cropped to maintain effect_size() aspect ratio
 * MUST be called from the render thread
 */
- (void)captureBlit:(int32_t)captureWidth
      captureHeight:(int32_t)captureHeight;

/**
 * Receive the last rendered frame in binary form.
 *
 * @return 4-byte per pixel data, size is `fx_width * fx_height * 4`
 * MUST be called from the render thread
 */
- (nonnull BNBPixelBuffer *)readPixels;

/** Thread-safe. May be called from any thread */
- (void)setRenderConsistencyMode:(BNBConsistencyMode)value;

/**
 * Process an image with current effect.
 *
 * Must be called from the render thread.
 *
 * @param inputImage to avoid conversion recommended to use YUV image
 * @param outputPixelFormat to avoid conversion recommended to use RGBA
 * @param params extra image processing arguments
 */
- (nonnull NSData *)processImage:(nonnull BNBFullImageData *)inputImage
               outputPixelFormat:(BNBPixelFormat)outputPixelFormat;

/**
 * Process an image with current effect.
 * Prefer this method over `processImage` when you have extra input data
 * besides just an input image.
 *
 * Must be called from render thread.
 *
 * @param inputFrameData `FrameData` with an image
 * @param outputPixelFormat to avoid conversion recommended to use RGBA
 * @param params extra image processing arguments
 */
- (nonnull NSData *)processImageFrameData:(nullable BNBFrameData *)inputFrameData
                        outputPixelFormat:(BNBPixelFormat)outputPixelFormat;

/**
 * Process image with current effect.
 * Must be called from render thread.
 * NOTE: inputImage have copy overhead, preferable to use process_image method
 * MUST be called from the render thread
 * @param inputImage input image with `width * height * 4` size
 * @param outputPixelFormat to avoid conversion recommended to use RGBA
 */
- (nonnull NSData *)processImageData:(nonnull NSData *)inputImage
                               width:(int32_t)width
                              height:(int32_t)height
                         orientation:(BNBCameraOrientation)orientation
                          isMirrored:(BOOL)isMirrored
                    inputPixelFormat:(BNBPixelFormat)inputPixelFormat
                   outputPixelFormat:(BNBPixelFormat)outputPixelFormat;

/**
 * Provides image to process and to play effect.
 * Thread-safe. May be called from any thread
 */
- (void)pushFrame:(nonnull BNBFullImageData *)fullImage;

/**
 * Provides image to process and to play effect.
 * Thread-safe. May be called from any thread
 */
- (void)pushFrameWithNumber:(nonnull BNBFullImageData *)fullImage
                frameNumber:(int64_t)frameNumber;

/**
 * Provides FrameData to process and to play effect.
 * Must contain full image.
 * Thread-safe. May be called from any thread
 */
- (void)pushFrameData:(nullable BNBFrameData *)frameData;

/**
 * Provides FrameData with frame number to process and to play effect.
 * Must contain full image.
 * Thread-safe. May be called from any thread
 */
- (void)pushFrameDataWithNumber:(nullable BNBFrameData *)frameData
                    frameNumber:(int64_t)frameNumber;

/** MUST be called from the main(render) thread */
- (void)playbackPlay;

- (void)playbackPause;

- (void)playbackStop;

/** Thread-safe. May be called from any thread */
- (BNBEffectPlayerPlaybackState)getPlaybackState;

/**
 * Get interface to control user iterations. This events will be passed to effect.
 * Thread-safe. May be called from any thread
 */
- (nullable BNBInputManager *)getInputManager;

/**
 * Initialize video processing. To provide frames use *process_video_frame* methods.
 * Effect audio is recorded as well and can be accessed using *process_recorded_audio*
 * and *write_recorded_audio*.
 * *push_frame* should not be called during processing.
 * MUST be called from the render thread
 * @throw Exception Must be aborted with stop_video_processing()
 */
- (void)startVideoProcessing:(int64_t)screenWidth
                screenHeight:(int64_t)screenHeight
                 orientation:(BNBCameraOrientation)orientation
                 resetEffect:(BOOL)resetEffect
                 offlineMode:(BOOL)offlineMode;

/**
 * Finish processing and return renderer to normal state.
 * MUST be called from the render thread
 */
- (void)stopVideoProcessing:(BOOL)resetEffect;

/**
 * Provide frame data to evaluate video processing.
 * MUST be called from the render thread
 * @param inputFrameData Frame Data to process.
 * @param params Processing params.
 * @param recognizerIterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
 * @return Frame data with frame processing results.
 */
- (nullable BNBFrameData *)processVideoFrameData:(nullable BNBFrameData *)inputFrameData
                            recognizerIterations:(nullable NSNumber *)recognizerIterations;

/**
 * Provide frame to evaluate video processing.
 * MUST be called from the render thread
 * @param inputImage Image to process.
 * @param params Processing params.
 * @param recognizerIterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
 * @return Frame data with frame processing results.
 */
- (nullable BNBFrameData *)processVideoFrame:(nonnull BNBFullImageData *)inputImage
                        recognizerIterations:(nullable NSNumber *)recognizerIterations;

/**
 * Draw processed frame to image buffer.
 * MUST be called from the render thread
 * @param inputFrameData FrameData to draw.
 * @param timeNs Frame position on timeline.
 * @param outputPixelFormat Output image format.
 * @return Buffer with processed image in selected format.
 */
- (nonnull NSData *)drawVideoFrame:(nullable BNBFrameData *)inputFrameData
                            timeNs:(int64_t)timeNs
                 outputPixelFormat:(BNBPixelFormat)outputPixelFormat;

/**
 * The same as *draw_video_frame* but accepts preallocated buffer to avoid recreation for every frame.
 * MUST be called from the render thread
 */
- (nonnull NSData *)drawVideoFrameAllocated:(nullable BNBFrameData *)inputFrameData
                                     timeNs:(int64_t)timeNs
                          outputPixelFormat:(BNBPixelFormat)outputPixelFormat
                            allocatedBuffer:(nonnull NSData *)allocatedBuffer;

/**
 * Save recorded audio.
 * Not thread-safe but can be called from any thread.
 * @param filename ".wav" extension must be used.
 * @param lengthMs Track length. Set to '0' to use full duration.
 */
- (void)writeRecordedAudio:(nonnull NSString *)filename
                  lengthMs:(int64_t)lengthMs;

/**
 * Trigger record start events in EP and Effect.
 * Records all sounds during recording
 * to be replayed by `processRecordedAudio`.
 * If @param playAudioWhileRecording is true, the audio will continue to play. Otherwise, the audio will be muted. 
 * Thread-safe. May be called from any thread
 */
- (void)onVideoRecordStart:(BOOL)playAudioWhileRecording;

/**
 * Triggers record stop events in EP and Effect.
 * Thread-safe. May be called from any thread
 */
- (void)onVideoRecordEnd;

/**
 * Get effect manager object
 * Thread-safe. May be called from any thread
 */
- (nullable BNBEffectManager *)effectManager;

/**
 * Force recognizer offline mode
 * Thread-safe. May be called from any thread
 */
- (void)setRecognizerOfflineMode:(BOOL)on;

/**
 * Set future frame filtrarion mode.
 * Produce smoother recognition result (anti jitter), however adds inconsistency in push'ed/draw'ed frames (one frame lag)
 * Example: push frame 1 - draw frame 1, push frame 2 - draw frame 1, push frame 3 - draw frame 2, ...
 */
- (void)setRecognizerUseFutureFilter:(BOOL)on;

/**
 * Set future frame interpolation mode.
 * Produce faster recognition result (skip even frames), however adds inconsistency in push'ed/pop'ed frames (one frame lag)
 * Example: push frame 1 - pop frame 1, push frame 2 - pop frame 1, push frame 3 - pop frame 2, ...
 */
- (void)setRecognizerUseFutureInterpolate:(BOOL)on;

/**
 * Set frame processor as current
 * Thread-safe. May be called from any thread
 */
- (void)setFrameProcessor:(nullable BNBFrameProcessor *)processor;

/**
 * Get current frame processor
 * Thread-safe. May be called from any thread
 */
- (nullable BNBFrameProcessor *)frameProcessor;

+ (void)setRenderBackend:(BNBRenderBackendType)backendType;

+ (BNBRenderBackendType)getCurrentRenderBackendType;

@end

Swift

class BNBEffectPlayer : NSObject