input.hpp

// Copyright (c) 2012-2017 VideoStitch SAS
// Copyright (c) 2018 stitchEm

#pragma once

#include "config.hpp"
#include "status.hpp"
#include "audio.hpp"
#include "frame.hpp"
#include "imuData.hpp"
#include "orah/exposureData.hpp"
#include "sink.hpp"

#include <stdint.h>
#include <cstdlib>
#include <limits>
#include <iosfwd>
#include <memory>
#include <mutex>

namespace VideoStitch {

namespace GPU {

template <typename T>
class Buffer;
class Surface;
class Stream;
}  // namespace GPU

namespace Input {

class VideoReader;
class AudioReader;
class MetadataReader;
class SinkReader;

enum class ReadStatusCode {
  // generic states
  Ok,
  ErrorWithStatus,
  // custom reader states
  EndOfFile,
  // the call is marked non-blocking and the requested operation would block, or:
  // there's no data available yet
  TryAgain,
};

typedef Result<ReadStatusCode> ReadStatus;

/**
 * @brief The base pointer for audio and video readers.
 */
class VS_EXPORT Reader {
 public:
  virtual ~Reader();

  // The index of this reader in the InputDefinition list of the project.
  // Makes the bond between the configuration values and the realtime objects.
  const readerid_t id;  ///< id

  /**
   * Type-casting to video reader
   * Might return null for readers with restricted capabilities.
   */
  VideoReader* getVideoReader() const;
  /**
   * Type-casting to audio reader
   * Might return null for readers with restricted capabilities.
   */
  AudioReader* getAudioReader() const;

  MetadataReader* getMetadataReader() const;

  SinkReader* getSinkReader() const;

  mtime_t getLatency();
  void setLatency(mtime_t); /* force new latency value */
  /*
   * update latency only if it increase and return true
   * return false if latency unmodified
   */
  bool updateLatency(mtime_t);

 protected:
  /**
   * @param id id
   */
  explicit Reader(readerid_t id);

 private:
  mtime_t latency;
  std::mutex latencyMutex;
};

/**
 * @brief The common interface to be implemented by all input video readers.
 *
 * Note that this class is performance-critical, so if you think about implementing
 * a custom reader you should be aware of general computing performance issues and read
 * the design docs of the input library.
 *
 * In particular, most reader will be I/O bound, and therefore the design is such
 * that it's better to do heavy processing in:
 *  - initialization methods
 *  - readFrame()
 *
 * Any other method is usually trivial.
 *
 * Note that readers should avoid being asynchronous and doing work in threads since the lib
 * already handles handles that at a higher level and strives to reuse buffers to limit memory usage.
 *
 * The pipeline works as follow: first, the frame data, with size getFrameDataSize(),
 * is read using readFrame() and transmitted to the GPU. readFrame() should never write
 * more than getFrameDataSize() bytes into the output buffer.
 * Loading immediately begins for the next image in parallel.
 * At the same time, the getFrameDataSize() of data is transmitted to the GPU.
 * If the pixel format is unknown to VideoStitch, unpackDevBuffer() is then called on the GPU device buffer data.
 * At this point, the data must be in packed 32-bit RGBA form on the GPU.
 * Then the reader is left alone and the mapping is started.
 *
 * In particular, procedural readers should strive to do all the work on the GPU
 * (i.e. in unpackDevBuffer).
 *
 * NOTE: By convention, frames are zero-based.
 */
class VS_EXPORT VideoReader : public virtual Reader {
 public:
  /**
   * @brief A class that holds time-constant specifications.
   */
  struct VS_EXPORT Spec {
    /**
     * Constructs a spec.
     * @param width The reader width.
     * @param height The reader height.
     * @param frameDataSize Size in bytes of a single frame.
     * @param pixelFormat The pixel format.
     * @param frameNum The number of frames for this reader.
     * @param frameRate Frame rate, in frames per second. Negative means unknown.
     * @param frameRateIsProcedural True if the frame rate is procedural, False otherwise, meaning it does not come from
     * an actual reader that needs to work with other readers sharing the same frame rate.
     * @param maskHostBuffer Mask data.
     * @param flags Reserved. Must be 0.
     */
    Spec(int64_t width, int64_t height, int64_t frameDataSize, VideoStitch::PixelFormat pixelFormat,
         AddressSpace addressspace, int frameNum, FrameRate frameRate, bool frameRateIsProcedural,
         const unsigned char* maskHostBuffer, int flags = 0);

    Spec();

    /**
     * Copy from @a spec.
     */
    Spec(const Spec& spec);

    ~Spec();

    /**
     * The reader width.
     */
    const int64_t width;
    /**
     * The reader height.
     */
    const int64_t height;
    /**
     * Size in bytes of a single frame.
     */
    const int64_t frameDataSize;
    /**
     * The pixel format.
     */
    const VideoStitch::PixelFormat format;
    /**
     * Input frames location.
     */
    const AddressSpace addressSpace;
    /**
     * Number of frames for this reader.
     */
    const int frameNum;
    /**
     * Frame rate, in frames per second.
     * Negative values mean that the reader has no known frame rate.
     */
    const FrameRate frameRate;
    /**
     * Flag whether the frame rate is procedural or not
     * meaning it does not come from an actual reader that needs to work with other readers sharing the same frame rate
     */
    const bool frameRateIsProcedural;
    /**
     * Flags. Reserved.
     */
    const int flags;
    /**
     * Host buffer containing input mask, of size width * height.
     * NULL means no mask. Not owned.
     */
    const unsigned char* const maskHostBuffer;

    /**
     * Returns a display name for the reader. Never assume any format for that. EVER.
     * That means that the only thing that's allowed with that is to diplay it to the user.
     * No testing for equality, parsing...
     * Thread safe.
     * @param os Sink for the display name.
     */
    void getDisplayName(std::ostream& os) const;

    /**
     * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small.
     * @param name The display name to set.
     */
    void setDisplayName(const char* name);

   private:
    Spec& operator=(const Spec&) = delete;

    class Impl;
    Impl* const pimpl;
  };

 public:
  virtual ~VideoReader();

  /**
   * Read a frame into videoFrame and advance to the next frame.
   * Optionally read an audio packet synchronized with this frame.
   * It is the Reader's job to ensure correct synchronization.
   *
   * @param date The capture/decode timestamp in microseconds.
   * @param videoFrame Where to write the video frame.
   *                   The provided buffer is of size at least getFrameDataSize().
   * @return a success status.
   */
  virtual ReadStatus readFrame(mtime_t& date, unsigned char* videoFrame) = 0;

  /**
   * Seek to the given frame.
   * @param frame Where to seek. Starts at the reader's first frame.
   * @return False on failure. In that case, the reader is free to leave the
   *                           current frame is in an undetermined state.
   *                           The client can call getCurFrame() or call
   *                           seekFrame() again to seek to another frame.
   * @note for implementors: On failure, it's advised to do the most efficient
   *                         thing given that the next call will probably be a
   *                         seek to the previous 'current' frame.
   */
  virtual Status seekFrame(frameid_t frame) = 0;

  /**
   * Returns the first frame in the sequence (inclusive).
   */
  frameid_t getFirstFrame() const;
  /**
   * Returns the last frame in the sequence (inclusive), or NO_LAST_FRAME.
   */
  frameid_t getLastFrame() const;

  /**
   * Convert the device memory from the internal pixel format to RGBA.
   * Use this function if your PixelFormat is Unknown to VideoStitch.
   *
   * Note that to be efficient, implementations of this method are not allowed
   * to do I/O. It is even encouraged that this method only contains a kernel call.
   * Implementations of this functions must be stateless (i.e. they cannot touch any
   * non-const state in the class).
   * The function should return only when the data in the buffer is ready to be
   * processed, or push a kernel call to the given CUDA stream.
   * The kernel call should use stream (cudaStream_t)stream, and should be asynchronous
   * for better performance.
   * @param dst Destination buffer
   * @param src source buffer
   * @param stream GPU stream on which the computations will run.
   */
  virtual Status unpackDevBuffer(GPU::Surface& dst, const GPU::Buffer<const unsigned char>& src,
                                 GPU::Stream& stream) const;

  /**
   * Returns the width of the reader.
   */
  int64_t getWidth() const;

  /**
   * Returns the height of the reader.
   */
  int64_t getHeight() const;

  /**
   * Returns the size of the data for one frame that
   * should be transfered over to the GPU.
   */
  int64_t getFrameDataSize() const;

  /**
   * Returns the (const) reader spec.
   */
  const Spec& getSpec() const;

  /**
   * Returns the reader spec.
   */
  Spec& getSpec();

  /**
   * Some readers need to allocate GPU memory behind the scene. Since GPU allocations are tied to threads,
   * any allocation that is done by the reader (including, but not restricted to, readFrame() or unpackDevBuffer())
   * will need to be done in the same thread that called these methods.
   * The controller will take care of this when the stitchers are released if implementors cleanup memory in this
   * method. You can skip implementing this method if you don't do such allocations.
   */
  virtual Status perThreadInit();

  /**
   * Performs per-thread cleanup. See perThreadInit().
   */
  virtual void perThreadCleanup();

  /**
   * Convert the device memory from a given known pixel format to RGBA.
   *
   * @param fmt Current pixel format of src Buffer, to be unpacked to RGBA.
   * @param dst Destination buffer
   * @param src read-only source buffer
   * @param width Frame width
   * @param height Frame height
   * @param stream GPU stream used for computations.
   */
  static Status unpackDevBuffer(VideoStitch::PixelFormat fmt, GPU::Surface& dst,
                                const GPU::Buffer<const unsigned char>& src, uint64_t width, uint64_t height,
                                GPU::Stream& stream);

 protected:
  /**
   * Constructor.
   * @param width The reader width. Readers cannot be resized.
   * @param height The reader height. Readers cannot be resized.
   * @param frameDataSize Size in pixels.
   * @param format Native pixel format of the Reader before unpacking.
   * @param addressSpace Address space of the produced frames.
   * @param frameRate Frame rate, in frames per second. Negative means unknown.
   * @param firstFrame First available frame.
   * @param lastFrame Last available frame.
   * @param isProcedural boolean whether the reader is procedural or not
   * @param maskHostBuffer Mask data.
   * @param flags Reserved. Must be 0.
   * @note Everything must be known at creation time.
   */
  VideoReader(int64_t width, int64_t height, int64_t frameDataSize, VideoStitch::PixelFormat format,
              AddressSpace addressSpace, FrameRate frameRate, frameid_t firstFrame, frameid_t lastFrame,
              bool isProcedural, const unsigned char* maskHostBuffer, int flags = 0);

 private:
  VideoReader();
  explicit VideoReader(const VideoReader&);

  /**
   * First available frame.
   */
  const unsigned firstFrame;

  /**
   * Last available frame.
   */
  const unsigned lastFrame;

  Spec spec;
};

/**
 * @brief The common interface to be implemented by all input audio readers.
 */
class VS_EXPORT AudioReader : public virtual Reader {
 public:
  /**
   * @brief A class that holds time-constant specifications.
   */
  struct VS_EXPORT Spec {
    /**
     * Constructs a spec.
     * @param layout Audio channels layout.
     * @param sampleRate Audio sampling frequency.
     * @param sampleDepth Format of the audio samples.
     */
    Spec(Audio::ChannelLayout layout, Audio::SamplingRate sampleRate, Audio::SamplingDepth sampleDepth);

    Spec();

    /**
     * Copy from @a spec.
     */
    Spec(const Spec& spec);

    ~Spec();

    /**
     * Audio channels layout
     */
    Audio::ChannelLayout layout;

    /**
     * Audio sampling rate.
     */
    Audio::SamplingRate sampleRate;

    /**
     * Audio channel depth and layout.
     */
    Audio::SamplingDepth sampleDepth;

    /**
     * Returns a display name for the reader. Never assume any format for that. EVER.
     * That means that the only thing that's allowed with that is to diplay it to the user.
     * No testing for equality, parsing...
     * Thread safe.
     * @param os Sink for the display name.
     */
    void getDisplayName(std::ostream& os) const;

    /**
     * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small.
     * @param name The display name to set.
     */
    void setDisplayName(const char* name);

   private:
    Spec& operator=(const Spec&) = delete;

    class Impl;
    Impl* const pimpl;
  };

 public:
  virtual ~AudioReader();

  /**
   * Read a frame of audio samples and advance to the next frame.
   *
   * @param nbSamples    Read a maximum of 'nbSamples' samples of audio into buffer.
   * @param audioSamples Where the audio samples channel planes will be written.
   *                     Up to 16 planar channels are supported.
   *                     Audio channel samples are either interleaved into a sample
   *                     frame or planar, either contiguous or padded (see Spec).
   *                     The memory of each plane is not allocated.
   *                     The ownership is transfered to the stitcher.
   *                     Also contains the timestamp of the first sample of the
   *                     buffer in microseconds.
   * @return a success status.
   */
  virtual ReadStatus readSamples(size_t nbSamples, Audio::Samples& audioSamples) = 0;

  /**
   * Seek to the given time.
   * @param date Where to seek.
   * @return False on failure. In that case, the reader is free to leave the
   *                           current frame is in an undetermined state.
   *                           The client can call getCurFrame() or call
   *                           seekFrame() again to seek to another frame.
   * @note for implementors: On failure, it's advised to do the most efficient
   *                         thing given that the next call will probably be a
   *                         seek to the previous 'current' frame.
   */
  virtual Status seekFrame(mtime_t date) = 0;

  /**
   * Returns the number of samples (per channel) currently available in the
   * reader's buffer.
   * @return number of samples (per channel) available to be read
   */
  virtual size_t available() = 0;

  /**
   * Used to determine if the end of stream has been reached.
   * @return true if the end of stream is reached, false otherwise.
   */
  virtual bool eos() = 0;

  /**
   * Returns the (const) reader spec.
   */
  const Spec& getSpec() const;

  /**
   * Returns the reader spec.
   */
  Spec& getSpec();

 protected:
  /**
   * Constructor.
   * @param layout Audio channels layout.
   * @param sampleRate Audio sample rate
   * @param sampleDepth Audio sample depth
   * @note Everything must be known at creation time.
   */
  AudioReader(Audio::ChannelLayout layout, Audio::SamplingRate sampleRate, Audio::SamplingDepth sampleDepth);

 private:
  AudioReader();
  explicit AudioReader(const AudioReader&);

  Spec spec;
};

static inline std::shared_ptr<AudioReader> getAudioSharedReader(std::shared_ptr<Reader> reader) {
  std::shared_ptr<AudioReader> audio = std::dynamic_pointer_cast<AudioReader>(reader);
  if (audio) {
    AudioReader::Spec spec = audio->getSpec();
    if (spec.layout != Audio::UNKNOWN && spec.sampleRate != Audio::SamplingRate::SR_NONE &&
        spec.sampleDepth != Audio::SamplingDepth::SD_NONE) {
      return audio;
    }
  }
  return nullptr;
}

static inline std::shared_ptr<VideoReader> getVideoSharedReader(std::shared_ptr<Reader> reader) {
  return std::dynamic_pointer_cast<VideoReader>(reader);
}

static inline std::shared_ptr<MetadataReader> getMetadataSharedReader(std::shared_ptr<Reader> reader) {
  return std::dynamic_pointer_cast<MetadataReader>(reader);
}

static inline std::shared_ptr<SinkReader> getSinkSharedReader(std::shared_ptr<Reader> reader) {
  return std::dynamic_pointer_cast<SinkReader>(reader);
}
/////////////////////////////

class VS_EXPORT MetadataReader : public virtual Reader {
 public:
  enum class MetadataReadStatusCode {
    // generic states
    Ok,
    ErrorWithStatus,
    // custom reader states
    EndOfFile,
    MoreDataAvailable,
  };

  typedef Result<MetadataReadStatusCode> MetadataReadStatus;

  /**
   * @brief A class that holds time-constant specifications.
   */
  struct VS_EXPORT Spec {
    /**
     * Constructs a spec.
     * @param framerate
     */
    explicit Spec(FrameRate framerate);

    Spec();

    /**
     * Copy from @a spec.
     */
    explicit Spec(const Spec& spec);

    ~Spec();

    /**
     * IMU frame rate
     */
    const FrameRate frameRate;

    /**
     * Returns a display name for the reader. Never assume any format for that. EVER.
     * That means that the only thing that's allowed with that is to diplay it to the user.
     * No testing for equality, parsing...
     * Thread safe.
     * @param os Sink for the display name.
     */
    void getDisplayName(std::ostream& os) const;

    /**
     * Reader implementors: use that to set the display name. You're free to put anything, and change it. Make it small.
     * @param name The display name to set.
     */
    void setDisplayName(const char* name);

   private:
    Spec& operator=(const Spec&) = delete;

    class Impl;
    Impl* const pimpl;
  };

 public:
  virtual ~MetadataReader();

  virtual Status readIMUSamples(std::vector<VideoStitch::IMU::Measure>& imuData) = 0;

  /**
   * Exposure, WhiteBalance, ToneCurve return values:
   * `Ok` if one sample was read, and there are no more samples available in the reader
   * `ErrorWithStatus` if a generic error was encountered
   * `EndOfFile` if the stream has stopped and there are no more samples available in the reader
   * `MoreDataAvailable` if one sample was read, and more data is available. the read function should be called again.
   */
  virtual MetadataReadStatus readExposure(std::map<videoreaderid_t, Metadata::Exposure>& exposure) = 0;
  virtual MetadataReadStatus readWhiteBalance(std::map<videoreaderid_t, Metadata::WhiteBalance>& whiteBalance) = 0;
  virtual MetadataReadStatus readToneCurve(std::map<videoreaderid_t, Metadata::ToneCurve>& toneCurve) = 0;

  /**
   * Returns the (const) reader spec.
   */
  const Spec& getSpec() const { return spec; }

  /**
   * Returns the reader spec.
   */
  Spec& getSpec() { return spec; }

 protected:
  /**
   * Constructor.
   * @param framerate
   * @note Everything must be known at creation time.
   */
  explicit MetadataReader(FrameRate framerate);

 private:
  MetadataReader();
  explicit MetadataReader(const MetadataReader&);

  Spec spec;
};

class VS_EXPORT SinkReader : public virtual IO::Sink<Reader> {
 protected:
  explicit SinkReader();
};
}  // namespace Input
}  // namespace VideoStitch