effect_player.hpp

Go to the documentation of this file.

/// \file
/// \addtogroup EffectPlayer
/// @{
///
// AUTOGENERATED FILE - DO NOT MODIFY!
// This file was generated by Djinni from effect_player.djinni
 
#pragma once
 
#include "bnb/scene/interfaces/render_backend_type.hpp"
#include "bnb/types/interfaces/frame_data.hpp"
#include "bnb/types/interfaces/pixel_buffer.hpp"
#include "bnb/types/interfaces/pixel_format.hpp"
#include <bnb/types/base_types.hpp>
#include <bnb/types/full_image.hpp>
#include <bnb/utils/defs.hpp>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <vector>
 
namespace bnb { namespace interfaces {
 
class camera_poi_listener;
class effect_activation_completion_listener;
class effect_info_listener;
class effect_manager;
class effect_player_configuration;
class face_number_listener;
class frame_data_listener;
class frame_duration_listener;
class frame_processor;
class input_manager;
enum class camera_orientation;
enum class consistency_mode;
enum class effect_player_playback_state;
 
/**
 * The EffectPlayer class provides ability to play AR effects on set of images or video.
 * <h2>Lifecycle</h2>
 * In order to support an application lifecycle EffectPlayer provides  special methods
 * to control it during application states like pause or losing focus.
 * <h3>Effect playback</h3>
 * Initial playback state of newly created Effect Player instance is `active`. With playback
 * control methods described below, the player can be launched/resumed or paused. State of effect
 * player becomes stopped/inactive at the moment of surface loss or by calling the
 * EffectPlayer::playback_stop method.
 * One can use next three methods to control effect playback:
 *   <br/> - `EffectPlayer.playback_play`. Switching to active state attempt. Possible from
 *   paused or stopped state and has no effect if effect playback is already active. Playback
 *   resumes from the position saved before `EffectPlayer.playbackPause` call.
 *   <br/> - `EffectPlayer.playback_pause`. Suspend current playback attempt. The recognizer thread
 *   is stopped and all the video textures and audio units playback is stopped as well. Effect
 *   player doesn't react on EffectPlayer::pushFrame calls in suspended state except
 *   asynchronous-inconsistent mode.
 *   <br/> - `EffectPlayer.playbackStop`. Switch to inactive state. In addition to pause clears
 *   recognizer's buffer. The next switch to active state will result in total rerun of active
 *   effect which means that it will be started from the very beginning.
 * <h2>Android usage example</h2>
 * If you want an application to be active on windowed mode and if the
 * focus is on another application you can switch EffectPlayer into active state when
 * `onStart` activity callback occurs and switch to paused when `onStop` occurs. Otherwise you
 * can suspend `EffectPlayer` on `onPause` and resume on `onResume` callback to make an
 * instance of the `EffectPlayer` inactive while activity losing focus.
 * <h2> Context/surface control</h2>
 * There are two methods that have to be used to ensure correct operating of the
 * EffectPlayer. Losing context without notifying the EffectPlayer
 * leads to application crash. Methods for context changes notification:
 *   <br/> - `EffectPlayer.surfaceCreated`
 *   <br/> - `EffectPlayer.surfaceDestroyed`
 * <h2> Multi-Threading performance and safety. Listeners.</h2>
 * EffectPlayer allows to call some of its methods from several threads (simultaneously) for convenience and increased performance.
 * This is allowed only during normal operation, all invocations from other threads MUST finish before
 * you start destroying the EffectPlayer instance.
 * Most methods belong to one of the two groups:
 *   <br/> - MUST be called from the "Main" (also "Render") thread ("Not Reentrant, Not Thread-safe")
 *   <br/> - MAY be called from "Any" thread. Most are "Thread-safe"
 * Listener callbacks can be called back from any thread.
 * Generally for best performance and responsiveness you'll have 3 threads:
 *   <br/> - Main(render) thread for drawing-related operations
 *   <br/> - "Camera" thread for handling and push()-ing frames into EffectPlayer
 *   <br/> - UI Thread for handling user interactions and other tasks
 */
class BNB_EXPORT effect_player {
public:
    virtual ~effect_player() {}
 
    static std::shared_ptr<effect_player> create(const std::shared_ptr<effect_player_configuration> & configuration);
 
    /**
     * Add callback to receive FPS information.
     * Thread-safe. May be called from any thread
     */
    virtual void add_frame_duration_listener(const std::shared_ptr<frame_duration_listener> & frame_duration_listener) = 0;
 
    /**
     * Remove callback to receive FPS information.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_frame_duration_listener(const std::shared_ptr<frame_duration_listener> & frame_duration_listener) = 0;
 
    /**
     * Add callback to receive faces count in frame.
     * Thread-safe. May be called from any thread
     */
    virtual void add_face_number_listener(const std::shared_ptr<face_number_listener> & face_number_listener) = 0;
 
    /**
     * Remove callback to receive faces count in frame.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_face_number_listener(const std::shared_ptr<face_number_listener> & face_number_listener) = 0;
 
    /**
     * Add callback to receive frame data right after processing in recognizer.
     * Thread-safe. May be called from any thread
     */
    virtual void add_frame_data_listener(const std::shared_ptr<frame_data_listener> & frame_data_listener) = 0;
 
    /**
     * Remove callback to receive frame data right after processing in recognizer.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_frame_data_listener(const std::shared_ptr<frame_data_listener> & frame_data_listener) = 0;
 
    /**
     * Add callback to receive center of the face in frame relative to top-left
     * corner in [0, 1] space.
     * Thread-safe. May be called from any thread
     */
    virtual void add_camera_poi_listener(const std::shared_ptr<camera_poi_listener> & camera_poi_listener) = 0;
 
    /**
     * Remove callback to receive center of the face in frame relative to top-left
     * Thread-safe. May be called from any thread
     */
    virtual void remove_camera_poi_listener(const std::shared_ptr<camera_poi_listener> & camera_poi_listener) = 0;
 
    /**
     * Add callback to receive current effect info from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void add_effect_info_listener(const std::shared_ptr<effect_info_listener> & effect_info_listener) = 0;
 
    /**
     * Remove callback to receive current effect info from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_effect_info_listener(const std::shared_ptr<effect_info_listener> & effect_info_listener) = 0;
 
    /**
     * Add callback to receive current effect activation notification from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void add_effect_activation_completion_listener(const std::shared_ptr<effect_activation_completion_listener> & effect_activation_completion_listener) = 0;
 
    /**
     * Remove callback to receive current effect activation notification from Effect Player.
     * Thread-safe. May be called from any thread
     */
    virtual void remove_effect_activation_completion_listener(const std::shared_ptr<effect_activation_completion_listener> & effect_activation_completion_listener) = 0;
 
    /**
     * Sets maximum allowed face results, if face tracking feature is present.
     * Thread-safe. May be called from any thread
     * This option is relevant only for GLFX effects.
     */
    virtual void set_max_faces(int32_t max_faces) = 0;
 
    /**
     * Set OpenGL name of input external texture.
     * https://developer.android.com/reference/android/graphics/SurfaceTexture .
     * MUST be called from the render thread
     *
     * @param name Texture name
     * @param width Textute width
     * @param height Texture height
     *
     * @see setUseExtCamTex
     *
     * <br/> Note: This function is Android only.
     * <br/> Note: We use texture size to calculate aspect ratio only.
     */
    virtual void set_external_camera_texture(int32_t name, int32_t width, int32_t height) = 0;
 
    /**
     * Use external texture as input source.
     * MUST be called from the render thread
     * @see setExternalCameraTexture
     * @note This function is Android only
     */
    virtual void set_use_ext_cam_tex(bool value) = 0;
 
    /**
     * Use to notify the EffectPlayer that the surface exists and effect can be played.
     * 0, 0 for width and height are valid values.
     * MUST be called from the render thread
     */
    virtual void surface_created(int32_t width, int32_t height) = 0;
 
    /**
     * Notify about rendering surface being resized.
     * MUST be called from the render thread
     */
    virtual void surface_changed(int32_t width, int32_t height) = 0;
 
    /**
     * This method should be called right before an active context will become invalid.
     * Switches playback state to inactive state. If it's not done an application will be
     * crashed on next draw iteration. After losing the surface effect playback can't be
     * resumed from last position.
     * MUST be called from the render thread
     */
    virtual void surface_destroyed() = 0;
 
    /**
     * Draw the current effect into the current OpenGL framebuffer. Uses internal frame_data
     * object obtained from latest push_frame recognition result.
     *
     * Return current frame number if drawing was performed and caller should swap buffers
     * otherwise `DRAW_SKIPPED`(-1)
     * MUST be called from the render thread
     */
    virtual int64_t draw() = 0;
 
    /**
     * Draw the current effect into the current OpenGL framebuffer. Uses externally provided frame_data
     * object instead of internal one obtained from latest push_frame recognition result.
     *
     * Return frame number from provided frame_data if drawing was performed and caller should swap buffers
     * otherwise `DRAW_SKIPPED`(-1)
     * MUST be called from the render thread
     */
    virtual int64_t draw_with_external_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & frame_data) = 0;
 
    /**
     * Record last rendering ("draw_()") result into current OpenGL framebuffer.
     * Content is cropped to maintain effect_size() aspect ratio
     * MUST be called from the render thread
     */
    virtual void capture_blit(int32_t capture_width, int32_t capture_height) = 0;
 
    /**
     * Receive the last rendered frame in binary form.
     *
     * @return 4-byte per pixel data, size is `fx_width * fx_height * 4`
     * MUST be called from the render thread
     */
    virtual ::bnb::interfaces::pixel_buffer read_pixels() = 0;
 
    /** Thread-safe. May be called from any thread */
    virtual void set_render_consistency_mode(consistency_mode value) = 0;
 
    /**
     * Process an image with current effect.
     *
     * Must be called from the render thread.
     *
     * @param input_image to avoid conversion recommended to use YUV image
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     * @param params extra image processing arguments
     */
    virtual ::bnb::data_t process_image(::bnb::full_image_t input_image, ::bnb::interfaces::pixel_format output_pixel_format) = 0;
 
    /**
     * Process an image with current effect.
     * Prefer this method over `processImage` when you have extra input data
     * besides just an input image.
     *
     * Must be called from render thread.
     *
     * @param input_frame_data `FrameData` with an image
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     * @param params extra image processing arguments
     */
    virtual ::bnb::data_t process_image_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, ::bnb::interfaces::pixel_format output_pixel_format) = 0;
 
    /**
     * Process image with current effect.
     * Must be called from render thread.
     * NOTE: input_image have copy overhead, preferable to use process_image method
     * MUST be called from the render thread
     * @param input_image input image with `width * height * 4` size
     * @param output_pixel_format to avoid conversion recommended to use RGBA
     */
    virtual std::vector<uint8_t> process_image_data(const std::vector<uint8_t> & input_image, int32_t width, int32_t height, camera_orientation orientation, bool is_mirrored, ::bnb::interfaces::pixel_format input_pixel_format, ::bnb::interfaces::pixel_format output_pixel_format) = 0;
 
    /**
     * Provides image to process and to play effect.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame(::bnb::full_image_t full_image) = 0;
 
    /**
     * Provides image to process and to play effect.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame_with_number(::bnb::full_image_t full_image, int64_t frame_number) = 0;
 
    /**
     * Provides FrameData to process and to play effect.
     * Must contain full image.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & frame_data) = 0;
 
    /**
     * Provides FrameData with frame number to process and to play effect.
     * Must contain full image.
     * Thread-safe. May be called from any thread
     */
    virtual void push_frame_data_with_number(const std::shared_ptr<::bnb::interfaces::frame_data> & frame_data, int64_t frame_number) = 0;
 
    /** MUST be called from the main(render) thread */
    virtual void playback_play() = 0;
 
    virtual void playback_pause() = 0;
 
    virtual void playback_stop() = 0;
 
    /** Thread-safe. May be called from any thread */
    virtual effect_player_playback_state get_playback_state() = 0;
 
    /**
     * Get interface to control user iterations. This events will be passed to effect.
     * Thread-safe. May be called from any thread
     */
    virtual std::shared_ptr<input_manager> get_input_manager() = 0;
 
    /**
     * Initialize video processing. To provide frames use *process_video_frame* methods.
     * Effect audio is recorded as well and can be accessed using *process_recorded_audio*
     * and *write_recorded_audio*.
     * *push_frame* should not be called during processing.
     * MUST be called from the render thread
     * @throw Exception Must be aborted with stop_video_processing()
     */
    virtual void start_video_processing(int64_t screen_width, int64_t screen_height, camera_orientation orientation, bool reset_effect, bool offline_mode) = 0;
 
    /**
     * Finish processing and return renderer to normal state.
     * MUST be called from the render thread
     */
    virtual void stop_video_processing(bool reset_effect) = 0;
 
    /**
     * Provide frame data to evaluate video processing.
     * MUST be called from the render thread
     * @param input_frame_data Frame Data to process.
     * @param params Processing params.
     * @param recognizer_iterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
     * @return Frame data with frame processing results.
     */
    virtual std::shared_ptr<::bnb::interfaces::frame_data> process_video_frame_data(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, std::optional<int32_t> recognizer_iterations) = 0;
 
    /**
     * Provide frame to evaluate video processing.
     * MUST be called from the render thread
     * @param input_image Image to process.
     * @param params Processing params.
     * @param recognizer_iterations Number of processing iterations. Higher number means higher processing quality, but lower speed. Must be greater than 1. Pass null value for default number.
     * @return Frame data with frame processing results.
     */
    virtual std::shared_ptr<::bnb::interfaces::frame_data> process_video_frame(::bnb::full_image_t input_image, std::optional<int32_t> recognizer_iterations) = 0;
 
    /**
     * Draw processed frame to image buffer.
     * MUST be called from the render thread
     * @param input_frame_data FrameData to draw.
     * @param time_ns Frame position on timeline.
     * @param output_pixel_format Output image format.
     * @return Buffer with processed image in selected format.
     */
    virtual ::bnb::data_t draw_video_frame(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, int64_t time_ns, ::bnb::interfaces::pixel_format output_pixel_format) = 0;
 
    /**
     * The same as *draw_video_frame* but accepts preallocated buffer to avoid recreation for every frame.
     * MUST be called from the render thread
     */
    virtual ::bnb::data_t draw_video_frame_allocated(const std::shared_ptr<::bnb::interfaces::frame_data> & input_frame_data, int64_t time_ns, ::bnb::interfaces::pixel_format output_pixel_format, ::bnb::data_t allocated_buffer) = 0;
 
    /**
     * Save recorded audio.
     * Not thread-safe but can be called from any thread.
     * @param filename ".wav" extension must be used.
     * @param length_ms Track length. Set to '0' to use full duration.
     */
    virtual void write_recorded_audio(const std::string & filename, int64_t length_ms) = 0;
 
    /**
     * Trigger record start events in EP and Effect.
     * Records all sounds during recording
     * to be replayed by `processRecordedAudio`.
     * If @param play_audio_while_recording is true, the audio will continue to play. Otherwise, the audio will be muted. 
     * Thread-safe. May be called from any thread
     */
    virtual void on_video_record_start(bool play_audio_while_recording) = 0;
 
    /**
     * Triggers record stop events in EP and Effect.
     * Thread-safe. May be called from any thread
     */
    virtual void on_video_record_end() = 0;
 
    /**
     * Get effect manager object
     * Thread-safe. May be called from any thread
     */
    virtual std::shared_ptr<::bnb::interfaces::effect_manager> effect_manager() = 0;
 
    /**
     * Force recognizer offline mode
     * Thread-safe. May be called from any thread
     */
    virtual void set_recognizer_offline_mode(bool on) = 0;
 
    /**
     * Set future frame filtrarion mode.
     * Produce smoother recognition result (anti jitter), however adds inconsistency in push'ed/draw'ed frames (one frame lag)
     * Example: push frame 1 - draw frame 1, push frame 2 - draw frame 1, push frame 3 - draw frame 2, ...
     */
    virtual void set_recognizer_use_future_filter(bool on) = 0;
 
    /**
     * Set future frame interpolation mode.
     * Produce faster recognition result (skip even frames), however adds inconsistency in push'ed/pop'ed frames (one frame lag)
     * Example: push frame 1 - pop frame 1, push frame 2 - pop frame 1, push frame 3 - pop frame 2, ...
     */
    virtual void set_recognizer_use_future_interpolate(bool on) = 0;
 
    /**
     * Set frame processor as current
     * Thread-safe. May be called from any thread
     */
    virtual void set_frame_processor(const std::shared_ptr<::bnb::interfaces::frame_processor> & processor) = 0;
 
    /**
     * Get current frame processor
     * Thread-safe. May be called from any thread
     */
    virtual std::shared_ptr<::bnb::interfaces::frame_processor> frame_processor() = 0;
 
    static void set_render_backend(::bnb::interfaces::render_backend_type backend_type);
 
    static ::bnb::interfaces::render_backend_type get_current_render_backend_type();
};
 
} }  // namespace bnb::interfaces
/// @}