// Copyright (c) 2012-2017 VideoStitch SAS // Copyright (c) 2018 stitchEm #pragma once #include "config.hpp" #include "status.hpp" #include "audio.hpp" #include "frame.hpp" #include "imuData.hpp" #include "orah/exposureData.hpp" #include "sink.hpp" #include #include #include #include #include #include namespace VideoStitch { namespace GPU { template class Buffer; class Surface; class Stream; } // namespace GPU namespace Input { class VideoReader; class AudioReader; class MetadataReader; class SinkReader; enum class ReadStatusCode { // generic states Ok, ErrorWithStatus, // custom reader states EndOfFile, // the call is marked non-blocking and the requested operation would block, or: // there's no data available yet TryAgain, }; typedef Result ReadStatus; /** * @brief The base pointer for audio and video readers. */ class VS_EXPORT Reader { public: virtual ~Reader(); // The index of this reader in the InputDefinition list of the project. // Makes the bond between the configuration values and the realtime objects. const readerid_t id; ///< id /** * Type-casting to video reader * Might return null for readers with restricted capabilities. */ VideoReader* getVideoReader() const; /** * Type-casting to audio reader * Might return null for readers with restricted capabilities. */ AudioReader* getAudioReader() const; MetadataReader* getMetadataReader() const; SinkReader* getSinkReader() const; mtime_t getLatency(); void setLatency(mtime_t); /* force new latency value */ /* * update latency only if it increase and return true * return false if latency unmodified */ bool updateLatency(mtime_t); protected: /** * @param id id */ explicit Reader(readerid_t id); private: mtime_t latency; std::mutex latencyMutex; }; /** * @brief The common interface to be implemented by all input video readers. * * Note that this class is performance-critical, so if you think about implementing * a custom reader you should be aware of general computing performance issues and read * the design docs of the input library. * * In particular, most reader will be I/O bound, and therefore the design is such * that it's better to do heavy processing in: * - initialization methods * - readFrame() * * Any other method is usually trivial. * * Note that readers should avoid being asynchronous and doing work in threads since the lib * already handles handles that at a higher level and strives to reuse buffers to limit memory usage. * * The pipeline works as follow: first, the frame data, with size getFrameDataSize(), * is read using readFrame() and transmitted to the GPU. readFrame() should never write * more than getFrameDataSize() bytes into the output buffer. * Loading immediately begins for the next image in parallel. * At the same time, the getFrameDataSize() of data is transmitted to the GPU. * If the pixel format is unknown to VideoStitch, unpackDevBuffer() is then called on the GPU device buffer data. * At this point, the data must be in packed 32-bit RGBA form on the GPU. * Then the reader is left alone and the mapping is started. * * In particular, procedural readers should strive to do all the work on the GPU * (i.e. in unpackDevBuffer). * * NOTE: By convention, frames are zero-based. */ class VS_EXPORT VideoReader : public virtual Reader { public: /** * @brief A class that holds time-constant specifications. */ struct VS_EXPORT Spec { /** * Constructs a spec. * @param width The reader width. * @param height The reader height. * @param frameDataSize Size in bytes of a single frame. * @param pixelFormat The pixel format. * @param frameNum The number of frames for this reader. * @param frameRate Frame rate, in frames per second. Negative means unknown. * @param frameRateIsProcedural True if the frame rate is procedural, False otherwise, meaning it does not come from * an actual reader that needs to work with other readers sharing the same frame rate. * @param maskHostBuffer Mask data. * @param flags Reserved. Must be 0. */ Spec(int64_t width, int64_t height, int64_t frameDataSize, VideoStitch::PixelFormat pixelFormat, AddressSpace addressspace, int frameNum, FrameRate frameRate, bool frameRateIsProcedural, const unsigned char* maskHostBuffer, int flags = 0); Spec(); /** * Copy from @a spec. */ Spec(const Spec& spec); ~Spec(); /** * The reader width. */ const int64_t width; /** * The reader height. */ const int64_t height; /** * Size in bytes of a single frame. */ const int64_t frameDataSize; /** * The pixel format. */ const VideoStitch::PixelFormat format; /** * Input frames location. */ const AddressSpace addressSpace; /** * Number of frames for this reader. */ const int frameNum; /** * Frame rate, in frames per second. * Negative values mean that the reader has no known frame rate. */ const FrameRate frameRate; /** * Flag whether the frame rate is procedural or not * meaning it does not come from an actual reader that needs to work with other readers sharing the same frame rate */ const bool frameRateIsProcedural; /** * Flags. Reserved. */ const int flags; /** * Host buffer containing input mask, of size width * height. * NULL means no mask. Not owned. */ const unsigned char* const maskHostBuffer; /** * Returns a display name for the reader. Never assume any format for that. EVER. * That means that the only thing that's allowed with that is to diplay it to the user. * No testing for equality, parsing... * Thread safe. * @param os Sink for the display name. */ void getDisplayName(std::ostream& os) const; /** * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small. * @param name The display name to set. */ void setDisplayName(const char* name); private: Spec& operator=(const Spec&) = delete; class Impl; Impl* const pimpl; }; public: virtual ~VideoReader(); /** * Read a frame into videoFrame and advance to the next frame. * Optionally read an audio packet synchronized with this frame. * It is the Reader's job to ensure correct synchronization. * * @param date The capture/decode timestamp in microseconds. * @param videoFrame Where to write the video frame. * The provided buffer is of size at least getFrameDataSize(). * @return a success status. */ virtual ReadStatus readFrame(mtime_t& date, unsigned char* videoFrame) = 0; /** * Seek to the given frame. * @param frame Where to seek. Starts at the reader's first frame. * @return False on failure. In that case, the reader is free to leave the * current frame is in an undetermined state. * The client can call getCurFrame() or call * seekFrame() again to seek to another frame. * @note for implementors: On failure, it's advised to do the most efficient * thing given that the next call will probably be a * seek to the previous 'current' frame. */ virtual Status seekFrame(frameid_t frame) = 0; /** * Returns the first frame in the sequence (inclusive). */ frameid_t getFirstFrame() const; /** * Returns the last frame in the sequence (inclusive), or NO_LAST_FRAME. */ frameid_t getLastFrame() const; /** * Convert the device memory from the internal pixel format to RGBA. * Use this function if your PixelFormat is Unknown to VideoStitch. * * Note that to be efficient, implementations of this method are not allowed * to do I/O. It is even encouraged that this method only contains a kernel call. * Implementations of this functions must be stateless (i.e. they cannot touch any * non-const state in the class). * The function should return only when the data in the buffer is ready to be * processed, or push a kernel call to the given CUDA stream. * The kernel call should use stream (cudaStream_t)stream, and should be asynchronous * for better performance. * @param dst Destination buffer * @param src source buffer * @param stream GPU stream on which the computations will run. */ virtual Status unpackDevBuffer(GPU::Surface& dst, const GPU::Buffer& src, GPU::Stream& stream) const; /** * Returns the width of the reader. */ int64_t getWidth() const; /** * Returns the height of the reader. */ int64_t getHeight() const; /** * Returns the size of the data for one frame that * should be transfered over to the GPU. */ int64_t getFrameDataSize() const; /** * Returns the (const) reader spec. */ const Spec& getSpec() const; /** * Returns the reader spec. */ Spec& getSpec(); /** * Some readers need to allocate GPU memory behind the scene. Since GPU allocations are tied to threads, * any allocation that is done by the reader (including, but not restricted to, readFrame() or unpackDevBuffer()) * will need to be done in the same thread that called these methods. * The controller will take care of this when the stitchers are released if implementors cleanup memory in this * method. You can skip implementing this method if you don't do such allocations. */ virtual Status perThreadInit(); /** * Performs per-thread cleanup. See perThreadInit(). */ virtual void perThreadCleanup(); /** * Convert the device memory from a given known pixel format to RGBA. * * @param fmt Current pixel format of src Buffer, to be unpacked to RGBA. * @param dst Destination buffer * @param src read-only source buffer * @param width Frame width * @param height Frame height * @param stream GPU stream used for computations. */ static Status unpackDevBuffer(VideoStitch::PixelFormat fmt, GPU::Surface& dst, const GPU::Buffer& src, uint64_t width, uint64_t height, GPU::Stream& stream); protected: /** * Constructor. * @param width The reader width. Readers cannot be resized. * @param height The reader height. Readers cannot be resized. * @param frameDataSize Size in pixels. * @param format Native pixel format of the Reader before unpacking. * @param addressSpace Address space of the produced frames. * @param frameRate Frame rate, in frames per second. Negative means unknown. * @param firstFrame First available frame. * @param lastFrame Last available frame. * @param isProcedural boolean whether the reader is procedural or not * @param maskHostBuffer Mask data. * @param flags Reserved. Must be 0. * @note Everything must be known at creation time. */ VideoReader(int64_t width, int64_t height, int64_t frameDataSize, VideoStitch::PixelFormat format, AddressSpace addressSpace, FrameRate frameRate, frameid_t firstFrame, frameid_t lastFrame, bool isProcedural, const unsigned char* maskHostBuffer, int flags = 0); private: VideoReader(); explicit VideoReader(const VideoReader&); /** * First available frame. */ const unsigned firstFrame; /** * Last available frame. */ const unsigned lastFrame; Spec spec; }; /** * @brief The common interface to be implemented by all input audio readers. */ class VS_EXPORT AudioReader : public virtual Reader { public: /** * @brief A class that holds time-constant specifications. */ struct VS_EXPORT Spec { /** * Constructs a spec. * @param layout Audio channels layout. * @param sampleRate Audio sampling frequency. * @param sampleDepth Format of the audio samples. */ Spec(Audio::ChannelLayout layout, Audio::SamplingRate sampleRate, Audio::SamplingDepth sampleDepth); Spec(); /** * Copy from @a spec. */ Spec(const Spec& spec); ~Spec(); /** * Audio channels layout */ Audio::ChannelLayout layout; /** * Audio sampling rate. */ Audio::SamplingRate sampleRate; /** * Audio channel depth and layout. */ Audio::SamplingDepth sampleDepth; /** * Returns a display name for the reader. Never assume any format for that. EVER. * That means that the only thing that's allowed with that is to diplay it to the user. * No testing for equality, parsing... * Thread safe. * @param os Sink for the display name. */ void getDisplayName(std::ostream& os) const; /** * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small. * @param name The display name to set. */ void setDisplayName(const char* name); private: Spec& operator=(const Spec&) = delete; class Impl; Impl* const pimpl; }; public: virtual ~AudioReader(); /** * Read a frame of audio samples and advance to the next frame. * * @param nbSamples Read a maximum of 'nbSamples' samples of audio into buffer. * @param audioSamples Where the audio samples channel planes will be written. * Up to 16 planar channels are supported. * Audio channel samples are either interleaved into a sample * frame or planar, either contiguous or padded (see Spec). * The memory of each plane is not allocated. * The ownership is transfered to the stitcher. * Also contains the timestamp of the first sample of the * buffer in microseconds. * @return a success status. */ virtual ReadStatus readSamples(size_t nbSamples, Audio::Samples& audioSamples) = 0; /** * Seek to the given time. * @param date Where to seek. * @return False on failure. In that case, the reader is free to leave the * current frame is in an undetermined state. * The client can call getCurFrame() or call * seekFrame() again to seek to another frame. * @note for implementors: On failure, it's advised to do the most efficient * thing given that the next call will probably be a * seek to the previous 'current' frame. */ virtual Status seekFrame(mtime_t date) = 0; /** * Returns the number of samples (per channel) currently available in the * reader's buffer. * @return number of samples (per channel) available to be read */ virtual size_t available() = 0; /** * Used to determine if the end of stream has been reached. * @return true if the end of stream is reached, false otherwise. */ virtual bool eos() = 0; /** * Returns the (const) reader spec. */ const Spec& getSpec() const; /** * Returns the reader spec. */ Spec& getSpec(); protected: /** * Constructor. * @param layout Audio channels layout. * @param sampleRate Audio sample rate * @param sampleDepth Audio sample depth * @note Everything must be known at creation time. */ AudioReader(Audio::ChannelLayout layout, Audio::SamplingRate sampleRate, Audio::SamplingDepth sampleDepth); private: AudioReader(); explicit AudioReader(const AudioReader&); Spec spec; }; static inline std::shared_ptr getAudioSharedReader(std::shared_ptr reader) { std::shared_ptr audio = std::dynamic_pointer_cast(reader); if (audio) { AudioReader::Spec spec = audio->getSpec(); if (spec.layout != Audio::UNKNOWN && spec.sampleRate != Audio::SamplingRate::SR_NONE && spec.sampleDepth != Audio::SamplingDepth::SD_NONE) { return audio; } } return nullptr; } static inline std::shared_ptr getVideoSharedReader(std::shared_ptr reader) { return std::dynamic_pointer_cast(reader); } static inline std::shared_ptr getMetadataSharedReader(std::shared_ptr reader) { return std::dynamic_pointer_cast(reader); } static inline std::shared_ptr getSinkSharedReader(std::shared_ptr reader) { return std::dynamic_pointer_cast(reader); } ///////////////////////////// class VS_EXPORT MetadataReader : public virtual Reader { public: enum class MetadataReadStatusCode { // generic states Ok, ErrorWithStatus, // custom reader states EndOfFile, MoreDataAvailable, }; typedef Result MetadataReadStatus; /** * @brief A class that holds time-constant specifications. */ struct VS_EXPORT Spec { /** * Constructs a spec. * @param framerate */ explicit Spec(FrameRate framerate); Spec(); /** * Copy from @a spec. */ explicit Spec(const Spec& spec); ~Spec(); /** * IMU frame rate */ const FrameRate frameRate; /** * Returns a display name for the reader. Never assume any format for that. EVER. * That means that the only thing that's allowed with that is to diplay it to the user. * No testing for equality, parsing... * Thread safe. * @param os Sink for the display name. */ void getDisplayName(std::ostream& os) const; /** * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small. * @param name The display name to set. */ void setDisplayName(const char* name); private: Spec& operator=(const Spec&) = delete; class Impl; Impl* const pimpl; }; public: virtual ~MetadataReader(); virtual Status readIMUSamples(std::vector& imuData) = 0; /** * Exposure, WhiteBalance, ToneCurve return values: * `Ok` if one sample was read, and there are no more samples available in the reader * `ErrorWithStatus` if a generic error was encountered * `EndOfFile` if the stream has stopped and there are no more samples available in the reader * `MoreDataAvailable` if one sample was read, and more data is available. the read function should be called again. */ virtual MetadataReadStatus readExposure(std::map& exposure) = 0; virtual MetadataReadStatus readWhiteBalance(std::map& whiteBalance) = 0; virtual MetadataReadStatus readToneCurve(std::map& toneCurve) = 0; /** * Returns the (const) reader spec. */ const Spec& getSpec() const { return spec; } /** * Returns the reader spec. */ Spec& getSpec() { return spec; } protected: /** * Constructor. * @param framerate * @note Everything must be known at creation time. */ explicit MetadataReader(FrameRate framerate); private: MetadataReader(); explicit MetadataReader(const MetadataReader&); Spec spec; }; class VS_EXPORT SinkReader : public virtual IO::Sink { protected: explicit SinkReader(); }; } // namespace Input } // namespace VideoStitch