effect_player.hpp

Go to the documentation of this file.

/// \file
/// \addtogroup EffectPlayer
/// @{
///
// AUTOGENERATED FILE - DO NOT MODIFY!
// This file generated by Djinni from effect_player.djinni
 
#pragma once
 
#include "bnb/recognizer/interfaces/face_search_mode.hpp"
#include "bnb/scene/interfaces/scene.hpp"
#include "bnb/types/interfaces/frame_data.hpp"
#include "bnb/types/interfaces/pixel_format.hpp"
#include "bnb/types/interfaces/pixel_rect.hpp"
#include <bnb/types/base_types.hpp>
#include <bnb/types/full_image.hpp>
#include <bnb/utils/defs.hpp>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
 
namespace bnb { namespace interfaces {
 
class analytics_listener;
class camera_poi_listener;
class debug_interface;
class effect_activation_completion_listener;
class effect_info_listener;
class effect_manager;
class face_number_listener;
class frame_data_listener;
class frame_duration_listener;
class input_manager;
class low_light_listener;
enum class camera_orientation;
enum class consistency_mode;
enum class effect_player_playback_state;
struct analytics_config;
struct effect_player_configuration;
struct process_image_params;
 
/**
 * The EffectPlayer class provides ability to play AR effects on set of images or video.
 * <h2>Lifecycle</h2>
 * In order to support an application lifecycle EffectPlayer provides  special methods
 * to control it during application states like pause or losing focus.
 * <h3>Effect playback</h3>
 * Initial playback state of newly created Effect Player instance is `active`. With playback
 * control methods described below, the player can be launched/resumed or paused. State of effect
 * player becomes stopped/inactive at the moment of surface loss or by calling the
 * EffectPlayer::playback_stop method.
 * One can use next three methods to control effect playback:
 *   <br/> - `EffectPlayer.playback_play`. Switching to active state attempt. Possible from
 *   paused or stopped state and has no effect if effect playback is already active. Playback
 *   resumes from the position saved before `EffectPlayer.playbackPause` call.
 *   <br/> - `EffectPlayer.playback_pause`. Suspend current playback attempt. The recognizer thread
 *   is stopped and all the video textures and audio units playback is stopped as well. Effect
 *   player doesn't react on EffectPlayer::pushFrame calls in suspended state except
 *   asynchronous-inconsistent mode.
 *   <br/> - `EffectPlayer.playbackStop`. Switch to inactive state. In addition to pause clears
 *   recognizer's buffer. The next switch to active state will result in total rerun of active
 *   effect which means that it will be started from the very beginning.
 * <h2>Android usage example</h2>
 * If you want an application to be active on windowed mode and if the
 * focus is on another application you can switch EffectPlayer into active state when
 * `onStart` activity callback occurs and switch to paused when `onStop` occurs. Otherwise you
 * can suspend `EffectPlayer` on `onPause` and resume on `onResume` callback to make an
 * instance of the `EffectPlayer` inactive while activity losing focus.
 * <h2> Context/surface control</h2>
 * There are two methods that have to be used to ensure correct operating of the
 * EffectPlayer. Losing context without notifying the EffectPlayer
 * leads to application crash. Methods for context changes notification:
 *   <br/> - `EffectPlayer.surfaceCreated`
 *   <br/> - `EffectPlayer.surfaceDestroyed`
 * <h2> Multi-Threading performance and safety. Listeners.</h2>
 * EffectPlayer allows to call some of its methods from several threads (simultaneously) for convenience and increased performance.
 * This is allowed only during normal operation, all invocations from other threads MUST finish before
 * you start destroying the EffectPlayer instance.
 * Most methods belong to one of the two groups:
 *   <br/> - MUST be called from the "Main" (also "Render") thread ("Not Reentrant, Not Thread-safe")
 *   <br/> - MAY be called from "Any" thread. Most are "Thread-safe"
 * Listener callbacks can be called back from any thread.
 * Generally for best performance and responsiveness you'll have 3 threads:
 *   <br/> - Main(render) thread for drawing-related operations
 *   <br/> - "Camera" thread for handling and push()-ing frames into EffectPlayer
 *   <br/> - UI Thread for handling user interactions and other tasks
 */
class BNB_EXPORT effect_player {
public:
    virtual ~effect_player() {}
 
    static std::shared_ptr<effect_player> create(const effect_player_configuration & configuration);
 
    /**
     * Get major version of EffectPlayer. Use this method to filter out breaking changes in
     * implementation of this class.
     * @note this is not version of SDK.
     * @see EffectPlayer.versionMinor
     */
    static int32_t version_major();
 
    /**
     * Get minor version of EffectPlayer
     * @see EffectPlayer.versionMajor
     */
    static int32_t version_minor();
 
    /**
     * Add callback to receive FPS information.
     * Thread-safe. May be called from any thread
     */
    virtual void add_frame_duration_listener(const std::shared_ptr<frame_duration_listener> & frame_duration_listener) = 0;
 
    /**
     * Remove callback to receive FPS information.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_frame_duration_listener(const std::shared_ptr<frame_duration_listener> & frame_duration_listener) = 0;
 
    /**
     * Add callback to receive faces count in frame.
     * Thread-safe. May be called from any thread
     */
    virtual void add_face_number_listener(const std::shared_ptr<face_number_listener> & face_number_listener) = 0;
 
    /**
     * Remove callback to receive faces count in frame.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_face_number_listener(const std::shared_ptr<face_number_listener> & face_number_listener) = 0;
 
    /**
     * Add callback to receive frame data right after processing in recognizer.
     * Thread-safe. May be called from any thread
     */
    virtual void add_frame_data_listener(const std::shared_ptr<frame_data_listener> & frame_data_listener) = 0;
 
    /**
     * Remove callback to receive frame data right after processing in recognizer.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_frame_data_listener(const std::shared_ptr<frame_data_listener> & frame_data_listener) = 0;
 
    /**
     * Add callback to receive low light info
     * Thread-safe. May be called from any thread
     */
    virtual void add_low_light_listener(const std::shared_ptr<low_light_listener> & low_light_listener) = 0;
 
    /**
     * Remove callback to receive low light info
     * Thread-safe. May be called from any thread
     */
    virtual void remove_low_light_listener(const std::shared_ptr<low_light_listener> & low_light_listener) = 0;
 
    /**
     * Add callback to receive center of the face in frame relative to top-left
     * corner in [0, 1] space.
     * Thread-safe. May be called from any thread
     */
    virtual void add_camera_poi_listener(const std::shared_ptr<camera_poi_listener> & camera_poi_listener) = 0;
 
    /**
     * Remove callback to receive center of the face in frame relative to top-left
     * Thread-safe. May be called from any thread
     */
    virtual void remove_camera_poi_listener(const std::shared_ptr<camera_poi_listener> & camera_poi_listener) = 0;
 
    /**
     * Add callback to receive current effect info from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void add_effect_info_listener(const std::shared_ptr<effect_info_listener> & effect_info_listener) = 0;
 
    /**
     * Remove callback to receive current effect info from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_effect_info_listener(const std::shared_ptr<effect_info_listener> & effect_info_listener) = 0;
 
    /**
     * Add callback to receive current effect activation notification from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void add_effect_activation_completion_listener(const std::shared_ptr<effect_activation_completion_listener> & effect_activation_completion_listener) = 0;
 
    /**
     * Remove callback to receive current effect activation notification from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_effect_activation_completion_listener(const std::shared_ptr<effect_activation_completion_listener> & effect_activation_completion_listener) = 0;
 
    /**
     * Sets maximum allowed face results, if face tracking feature is present.
     * Thread-safe. May be called from any thread
     * This option is relevant only for GLFX effects.
     */
    virtual void set_max_faces(int32_t max_faces) = 0;
 
    /**
     * Set OpenGL name of input external texture.
     * https://developer.android.com/reference/android/graphics/SurfaceTexture .
     * MUST be called from the render thread
     *
     * @param name Texture name
     * @param width Textute width
     * @param height Texture height
     *
     * @see setUseExtCamTex
     *
     * <br/> Note: This function is Android only.
     * <br/> Note: We use texture size to calculate aspect ratio only.
     */
    virtual void set_external_camera_texture(int32_t name, int32_t width, int32_t height) = 0;
 
    /**
     * Use external texture as input source.
     * MUST be called from the render thread
     * @see setExternalCameraTexture
     * @note This function is Android only
     */
    virtual void set_use_ext_cam_tex(bool value) = 0;
 
    /**
     * Use to notify the EffectPlayer that the surface exists and effect can be played.
     * 0, 0 for width and height are valid values.
     * MUST be called from the render thread
     */
    virtual void surface_created(int32_t width, int32_t height) = 0;
 
    /**
     * Notify about rendering surface being resized.
     * MUST be called from the render thread
     */
    virtual void surface_changed(int32_t width, int32_t height) = 0;
 
    /**
     * This method should be called right before an active context will become invalid.
     * Switches playback state to inactive state. If it's not done an application will be
     * crashed on next draw iteration. After losing the surface effect playback can't be
     * resumed from last position.
     * MUST be called from the render thread
     */
    virtual void surface_destroyed() = 0;
 
    /**
     * Draw the current effect into the current OpenGL framebuffer. Uses internal frame_data
     * object obtained from latest push_frame recognition result.
     *
     * Return current frame number if drawing was performed and caller should swap buffers
     * otherwise `DRAW_SKIPPED`(-1)
     * MUST be called from the render thread
     */
    virtual int64_t draw() = 0;
 
    /**
     * Draw the current effect into the current OpenGL framebuffer. Uses externally provided frame_data
     * object instead of internal one obtained from latest push_frame recognition result.
     *
     * Return frame number from provided frame_data if drawing was performed and caller should swap buffers
     * otherwise `DRAW_SKIPPED`(-1)
     * MUST be called from the render thread
     */
    virtual int64_t draw_with_external_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & frame_data) = 0;
 
    /**
     * Record last rendering ("draw_()") result into current OpenGL framebuffer.
     * Content is cropped to maintain effect_size() aspect ratio
     * MUST be called from the render thread
     */
    virtual void capture_blit(int32_t capture_width, int32_t capture_height) = 0;
 
    /**
     * Receive the last rendered frame in binary form.
     *
     * @param width width of the picture to take, must match the values passed to
     * `setEffectSize` or `create`
     * @param height height of the picture to take, must match the values passed to
     * `setEffectSize` or `create`
     *
     * @return RGBA data, size is `width * height * 4`
     * MUST be called from the render thread
     */
    virtual ::bnb::data_t read_pixels(int32_t width, int32_t height) = 0;
 
    /** Thread-safe. May be called from any thread */
    virtual void set_render_consistency_mode(consistency_mode value) = 0;
 
    /**
     * Request display of sub-area of the input image into sub-area of the output surface, with optional x,y flips
     * image_rect is fitted inside viewport_rect
     * Resets transform to default if either rect has 0 dimensions
     * @param image_rect rectangle in input image coordinates(pixels) after applying input rotations and flips
     * @param viewport_rect rectangle in output surface coordinates(pixels)
     * MUST be called from the render thread
     */
    virtual void set_render_transform(const ::bnb::interfaces::pixel_rect & image_rect, const ::bnb::interfaces::pixel_rect & viewport_rect, bool x_flip, bool y_flip) = 0;
 
    /**
     * Process an image with current effect.
     *
     * Must be called from the render thread.
     *
     * @param input_image to avoid conversion recommended to use YUV image
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     * @param params extra image processing arguments
     */
    virtual ::bnb::data_t process_image(::bnb::full_image_t input_image, ::bnb::interfaces::pixel_format output_pixel_format, const process_image_params & params) = 0;
 
    /**
     * Process an image with current effect.
     * Prefer this method over `processImage` when you have extra input data
     * besides just an input image.
     *
     * Must be called from render thread.
     *
     * @param input_frame_data `FrameData` with an image
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     * @param params extra image processing arguments
     */
    virtual ::bnb::data_t process_image_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, ::bnb::interfaces::pixel_format output_pixel_format, const process_image_params & params) = 0;
 
    /**
     * Process image with current effect.
     * Must be called from render thread.
     * NOTE: input_image have copy overhead, preferable to use process_image method
     * MUST be called from the render thread
     * @param input_image input image with `width * height * 4` size
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     */
    virtual std::vector<uint8_t> process_image_data(const std::vector<uint8_t> & input_image, int32_t width, int32_t height, camera_orientation orientation, bool is_mirrored, ::bnb::interfaces::pixel_format input_pixel_format, ::bnb::interfaces::pixel_format output_pixel_format, const process_image_params & params) = 0;
 
    /**
     * Provides image to process and to play effect.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame(::bnb::full_image_t full_image) = 0;
 
    /**
     * Provides image to process and to play effect.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame_with_number(::bnb::full_image_t full_image, int64_t frame_number) = 0;
 
    /**
     * Provides FrameData to process and to play effect.
     * Must contain full image.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & frame_data) = 0;
 
    /** MUST be called from the main(render) thread */
    virtual void playback_play() = 0;
 
    virtual void playback_pause() = 0;
 
    virtual void playback_stop() = 0;
 
    /** Thread-safe. May be called from any thread */
    virtual effect_player_playback_state get_playback_state() = 0;
 
    /**
     * If currently loaded effect is based on new Scene engine and it is loaded completly returns Scene object from the effect.
     * Otherwise returns null.
     * MUST be called from the render thread
     * Returned Scene object MUST be used only from the render thread and only while the effect is in the loaded state.
     */
    virtual std::shared_ptr<::bnb::interfaces::scene> get_current_scene() = 0;
 
    /**
     * Set effect audio volume.
     * Thread-safe. May be called from any thread
     * @param volume A value in range `[0, 1]`, where `1` means maximum volume.
     */
    virtual void set_effect_volume(float volume) = 0;
 
    /** Set audio enabled */
    virtual void enable_audio(bool enable) = 0;
 
    /**
     * Get interface to control user iterations. This events will be passed to effect.
     * Thread-safe. May be called from any thread
     */
    virtual std::shared_ptr<input_manager> get_input_manager() = 0;
 
    /**
     * Initialize video processing. To provide frames use *process_video_frame* methods.
     * Effect audio is recorded as well and can be accessed using *process_recorded_audio*
     * and *write_recorded_audio*.
     * *push_frame* should not be called during processing.
     * MUST be called from the render thread
     * @throw Exception Must be aborted with stop_video_processing()
     */
    virtual void start_video_processing(int64_t screen_width, int64_t screen_height, camera_orientation orientation, bool reset_effect, bool offline_mode) = 0;
 
    /**
     * Finish processing and return renderer to normal state.
     * MUST be called from the render thread
     */
    virtual void stop_video_processing(bool reset_effect) = 0;
 
    /**
     * Provide frame data to evaluate video processing.
     * MUST be called from the render thread
     * @param input_frame_data Frame Data to process.
     * @param params Processing params.
     * @param recognizer_iterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
     * @return Frame data with frame processing results.
     */
    virtual std::shared_ptr<::bnb::interfaces::frame_data> process_video_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, const process_image_params & params, std::optional<int32_t> recognizer_iterations) = 0;
 
    /**
     * Provide frame to evaluate video processing.
     * MUST be called from the render thread
     * @param input_image Image to process.
     * @param params Processing params.
     * @param recognizer_iterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
     * @return Frame data with frame processing results.
     */
    virtual std::shared_ptr<::bnb::interfaces::frame_data> process_video_frame(::bnb::full_image_t input_image, const process_image_params & params, std::optional<int32_t> recognizer_iterations) = 0;
 
    /**
     * Draw processed frame to image buffer.
     * MUST be called from the render thread
     * @param input_frame_data FrameData to draw.
     * @param time_ns Frame position on timeline.
     * @param output_pixel_format Output image format.
     * @return Buffer with processed image in selected format.
     */
    virtual ::bnb::data_t draw_video_frame(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, int64_t time_ns, ::bnb::interfaces::pixel_format output_pixel_format) = 0;
 
    /**
     * The same as *draw_video_frame* but accepts preallocated buffer to avoid recreation for every frame.
     * MUST be called from the render thread
     */
    virtual ::bnb::data_t draw_video_frame_allocated(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, int64_t time_ns, ::bnb::interfaces::pixel_format output_pixel_format, ::bnb::data_t allocated_buffer) = 0;
 
    /** Thread-safe. May be called from any thread */
    virtual bool is_voice_changer_configured() = 0;
 
    /**
     * Process recorded voice with voice changer effects mixing with recorded sounds.
     * Copies in->out if no voice changer or effect is active.
     * Not thread-safe but can be called from any thread.
     * Recorded sounds have the same lifetime as the effect(e.g reset after load_effect, etc.).
     * @param in_filename Input filename.
     * @param out_filename Must have ".wav" extension.
     * @param mix_volume Relative volume for mixing in sounds, [0..1].
     */
    virtual void process_recorded_audio(const std::string & in_filename, const std::string & out_filename, float mix_volume) = 0;
 
    /**
     * Save recorded audio.
     * Not thread-safe but can be called from any thread.
     * @param filename ".wav" extension must be used.
     * @param length_ms Track length. Set to '0' to use full duration.
     */
    virtual void write_recorded_audio(const std::string & filename, int64_t length_ms) = 0;
 
    /**
     * Trigger record start events in EP and Effect.
     * Records all sounds during recording
     * to be replayed by `processRecordedAudio`.
     * If @param play_audio_while_recording is true, the audio will continue to play. Otherwise, the audio will be muted. 
     * Thread-safe. May be called from any thread
     */
    virtual void on_video_record_start(bool play_audio_while_recording) = 0;
 
    /**
     * Triggers record stop events in EP and Effect.
     * Thread-safe. May be called from any thread
     */
    virtual void on_video_record_end() = 0;
 
    /**
     * Check is device compatible with Neural Networks player
     * Thread-safe. May be called from any thread
     * On some platforms (e.g. Android) may require the first invocation to be on the render thread
     */
    virtual bool is_device_nn_compatible() = 0;
 
    /**
     * Request to start framedata capture process. Output file is in CBOR format.
     * Thread-safe. May be called from any thread
     * @param folder Output folder.
     * @param filename Output filename. If the value is an empty string, filename is generated based on
     *   date and time.
     */
    virtual void start_framedata_capture(const std::string & folder, const std::optional<std::string> & filename) = 0;
 
    /** @see startFramedataCapture */
    virtual void stop_framedata_capture() = 0;
 
    /**
     * Request to start analytics capture process.
     * Analytics supports two business cases:
     *   1. Capture high resolution photo with face (bigger than 720p).
     *   2. Capture N frames with face (N is configurable, if N=0 capturer will skip this case).
     * Result is a path to CBOR file with images and face recognition results, compressed with zlib.
     * Thread-safe. May be called from any thread
     * @param device_info Any user specific information to attach in analytics report.
     * @param config Setup configurable properties (like frames counter and delay before capture).
     * @param listener Callback which is fired in one of two above mentioned cases.
     */
    virtual void start_analytics_capture(const std::unordered_map<std::string, std::string> & device_info, const analytics_config & config, const std::shared_ptr<analytics_listener> & listener) = 0;
 
    virtual void stop_analytics_capture() = 0;
 
    /**
     * Get effect manager object
     * Thread-safe. May be called from any thread
     */
    virtual std::shared_ptr<::bnb::interfaces::effect_manager> effect_manager() = 0;
 
    /** For internal usage only */
    virtual std::shared_ptr<::bnb::interfaces::debug_interface> debug_interface() = 0;
 
    /**
     * Force recognizer offline mode
     * Thread-safe. May be called from any thread
     */
    virtual void set_recognizer_offline_mode(bool on) = 0;
 
    /**
     * Change face search mode
     * Thread-safe. May be called from any thread
     */
    virtual void set_face_search_mode(::bnb::interfaces::face_search_mode face_search) = 0;
};
 
} }  // namespace bnb::interfaces
/// @}