surface2.cpp

// Copyright (c) 2012-2017 VideoStitch SAS
// Copyright (c) 2018 stitchEm

#include "allocator.hpp"

#include "core/transformGeoParams.hpp"

#include "core1/panoRemapper.hpp"
#include "core1/imageMapping.hpp"
#include "core1/imageMerger.hpp"

#include "gpu/image/imageOps.hpp"

#include "libvideostitch/panoDef.hpp"
#include "libvideostitch/overlay.hpp"

//#define DUMP_FINAL
//#define DUMP_ORIGIN

#if defined(DUMP_FINAL) || defined DUMP_ORIGIN
#include "util/debugUtils.hpp"
#endif

namespace VideoStitch {
namespace Core {

SourceSurface::Pimpl::Pimpl(GPU::Surface* s, GPU::Stream stream) : surface(s), stream(stream) {}

SourceSurface::Pimpl::~Pimpl() {
  stream.destroy();
  delete surface;
}

PanoSurface::Pimpl::Pimpl(GPU::Stream stream, GPU::Buffer<uint32_t> buffer, size_t w, size_t h)
    : buffer(buffer), stream(stream), width(w), height(h) {}

PanoPimpl::PanoPimpl(GPU::Stream stream, GPU::Buffer<uint32_t> buffer, GPU::Surface* remap, size_t w, size_t h)
    : PanoSurface::Pimpl(stream, buffer, w, h), remapBuffer(remap) {}

CubemapPimpl::CubemapPimpl(bool equiangular, GPU::Stream stream, GPU::Buffer<uint32_t>* bufs,
                           GPU::Buffer<uint32_t> buffer, GPU::CubemapSurface* cubemap, GPU::Buffer<uint32_t> t,
                           size_t w)
    : CubemapSurface::Pimpl(stream, buffer, w), equiangular(equiangular), remapBuffer(cubemap), tmp(t) {
  for (int i = 0; i < 6; i++) {
    buffers[i] = bufs[i];
  }
}

PanoSurface::Pimpl::~Pimpl() {
  stream.destroy();
  if (!externalAlloc) {
    buffer.release();
  }
}

PanoPimpl::~PanoPimpl() { delete remapBuffer; }

CubemapPimpl::~CubemapPimpl() {
  if (!externalAlloc) {
    for (int i = 0; i < 6; i++) {
      buffers[i].release();
    }
  }

  delete remapBuffer;
  tmp.release();
}

Potential<SourceSurface::Pimpl> SourceSurface::Pimpl::create(GPU::Surface* surface) {
  PotentialValue<GPU::Stream> stream = GPU::Stream::create();
  if (stream.ok()) {
    return Potential<Pimpl>(new Pimpl(surface, stream.value()));
  } else {
    return Potential<Pimpl>(stream.status());
  }
}

void SourceSurface::acquire() { pimpl->acquireReader(); }

void SourceSurface::release() { pimpl->releaseReader(); }

void SourceSurface::Pimpl::acquireWriter() {
  std::unique_lock<std::mutex> lk(mutex);
  cv.wait(lk, [this] { return renderers == 0; });
  stitcher = true;
}

void SourceSurface::Pimpl::releaseWriter() {
  {
    std::lock_guard<std::mutex> lk(mutex);
    stitcher = false;
  }
  cv.notify_all();
}

void SourceSurface::Pimpl::acquireReader() {
  std::unique_lock<std::mutex> lk(mutex);
  cv.wait(lk, [this] { return !stitcher; });
  renderers++;
}

void SourceSurface::Pimpl::releaseReader() {
  {
    std::lock_guard<std::mutex> lk(mutex);
    renderers--;
  }
  cv.notify_one();
}

size_t SourceSurface::getWidth() const { return pimpl->getWidth(); }

size_t SourceSurface::getHeight() const { return pimpl->getHeight(); }

size_t SourceSurface::Pimpl::getWidth() const { return surface->width(); }

size_t SourceSurface::Pimpl::getHeight() const { return surface->height(); }

Potential<PanoPimpl> PanoPimpl::create(GPU::Buffer<uint32_t> buffer, GPU::Surface* surface, size_t w, size_t h) {
  PotentialValue<GPU::Stream> stream = GPU::Stream::create();
  if (stream.ok()) {
    return Potential<PanoPimpl>(new PanoPimpl(stream.value(), buffer, surface, w, h));
  } else {
    return Potential<PanoPimpl>(stream.status());
  }
}

void PanoSurface::acquire() { pimpl->acquireReader(); }

void PanoSurface::release() { pimpl->releaseReader(); }

void PanoSurface::Pimpl::acquireWriter() {
  std::unique_lock<std::mutex> lk(mutex);
  cv.wait(lk, [this] { return renderers == 0; });
  stitcher = true;
}

void PanoSurface::Pimpl::releaseWriter() {
  {
    std::lock_guard<std::mutex> lk(mutex);
    stitcher = false;
  }
  cv.notify_all();
}

void PanoSurface::Pimpl::acquireReader() {
  std::unique_lock<std::mutex> lk(mutex);
  cv.wait(lk, [this] { return !stitcher; });
  renderers++;
}

void PanoSurface::Pimpl::releaseReader() {
  {
    std::lock_guard<std::mutex> lk(mutex);
    renderers--;
  }
  cv.notify_one();
}

size_t PanoSurface::getWidth() const { return pimpl->getWidth(); }

size_t PanoSurface::getHeight() const { return pimpl->getHeight(); }

SourceSurface::SourceSurface(Pimpl* pimpl) : pimpl(pimpl) {}

SourceSurface::~SourceSurface() { delete pimpl; }

PanoSurface::PanoSurface(Pimpl* pimpl) : pimpl(pimpl) {}

PanoSurface::~PanoSurface() { delete pimpl; }

CubemapSurface::CubemapSurface(Pimpl* pimpl) : PanoSurface(pimpl) {}

CubemapSurface::~CubemapSurface() {}

size_t CubemapSurface::getLength() const { return dynamic_cast<CubemapPimpl*>(pimpl)->getLength(); }

Status PanoPimpl::reset(const Core::ImageMerger* merger) {
  if (merger && (merger->warpMergeType() == Core::ImageMerger::Format::Gradient)) {
    return memsetToZeroAsync(*remapBuffer, stream);
  } else {
    return memsetToZeroAsync(buffer, stream);
  }
}

Status PanoPimpl::reproject(const Core::PanoDefinition& pano, const Matrix33<double>& perspective,
                            const Core::ImageMerger* merger) {
#if defined(DUMP_ORIGIN)
  if (merger && (merger->warpMergeType() == Core::ImageMerger::Format::Gradient)) {
    memcpyAsync(buffer, *remapBuffer, stream);
  }
  stream.synchronize();
  Debug::dumpRGBADeviceBuffer("origin.png", buffer, (unsigned)pano.getWidth(), (unsigned)pano.getHeight());
#endif

  if (width != (unsigned)pano.getWidth() || height != (unsigned)pano.getHeight()) {
    return {Origin::Surface, ErrType::InvalidConfiguration, "Surface unadapted to the current panorama configuration"};
  }

  if (!(merger && (merger->warpMergeType() == Core::ImageMerger::Format::Gradient))) {
    memcpyAsync(*remapBuffer, buffer.as_const(), stream);
  }

  float2 srcScale = {
      Core::TransformGeoParams::computePanoScale(Core::PanoProjection::Equirectangular, pano.getWidth(), 360.f),
      2 * Core::TransformGeoParams::computePanoScale(Core::PanoProjection::Equirectangular, pano.getHeight(), 360.f)};
  float2 dstScale = {
      Core::TransformGeoParams::computePanoScale(pano.getProjection(), pano.getWidth(), (float)pano.getHFOV()),
      Core::TransformGeoParams::computePanoScale(pano.getProjection(), pano.getWidth(), (float)pano.getHFOV())};

  switch (pano.getProjection()) {
    case Core::PanoProjection::Rectilinear:
      return Core::reprojectRectilinear(buffer, dstScale, *remapBuffer, srcScale, (unsigned)pano.getWidth(),
                                        (unsigned)pano.getHeight(), perspective, stream);
    case Core::PanoProjection::Cylindrical:
      assert(false);
      return Status::OK();
    case Core::PanoProjection::Equirectangular:
      return Core::reprojectEquirectangular(buffer, dstScale, *remapBuffer, srcScale, (unsigned)pano.getWidth(),
                                            (unsigned)pano.getHeight(), perspective, stream);
    case Core::PanoProjection::FullFrameFisheye:
      return Core::reprojectFullFrameFisheye(buffer, dstScale, *remapBuffer, srcScale, (unsigned)pano.getWidth(),
                                             (unsigned)pano.getHeight(), perspective, stream);
    case Core::PanoProjection::CircularFisheye:
      return Core::reprojectCircularFisheye(buffer, dstScale, *remapBuffer, srcScale, (unsigned)pano.getWidth(),
                                            (unsigned)pano.getHeight(), perspective, stream);
    case Core::PanoProjection::Stereographic:
      return Core::reprojectStereographic(buffer, dstScale, *remapBuffer, srcScale, (unsigned)pano.getWidth(),
                                          (unsigned)pano.getHeight(), perspective, stream);
    case Core::PanoProjection::Cubemap:
    case Core::PanoProjection::EquiangularCubemap:
      assert(false);
      return Status::OK();
  }
  return Status::OK();
}

Status CubemapPimpl::reproject(const Core::PanoDefinition& pano, const Matrix33<double>& perspective,
                               const Core::ImageMerger*) {
#if defined(DUMP_ORIGIN)
  stream.synchronize();
  Debug::dumpRGBADeviceBuffer("origin_face_+x.png", buffers[0], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("origin_face_-x.png", buffers[1], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("origin_face_+y.png", buffers[2], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("origin_face_-y.png", buffers[3], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("origin_face_+z.png", buffers[4], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("origin_face_-z.png", buffers[5], (unsigned)pano.getLength(), (unsigned)pano.getLength());
#endif

  memcpyCubemapAsync(*remapBuffer, buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5],
                     pano.getLength(), stream);

  stream.synchronize();

  rotateCubemap(pano, *remapBuffer, buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5], perspective,
                equiangular, stream);

#if defined(DUMP_FINAL)
  stream.synchronize();
  Debug::dumpRGBADeviceBuffer("face_+x.png", buffers[0], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("face_-x.png", buffers[1], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("face_+y.png", buffers[2], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("face_-y.png", buffers[3], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("face_+z.png", buffers[4], (unsigned)pano.getLength(), (unsigned)pano.getLength());
  Debug::dumpRGBADeviceBuffer("face_-z.png", buffers[5], (unsigned)pano.getLength(), (unsigned)pano.getLength());
#endif

  return Status::OK();
}

Status PanoPimpl::warp(Core::ImageMapping* mapping, frameid_t frame, const Core::PanoDefinition& pano,
                       GPU::Stream& stream) {
  // warp the image to the destination space
  if (mapping->getMerger().isMultiScale()) {
    FAIL_RETURN(mapping->warp(frame, pano, progressivePbo.borrow(), *remapBuffer, stream));
  } else {
    FAIL_RETURN(mapping->warp(frame, pano, buffer, *remapBuffer, stream));
  }

  // analyze the image content if needed (eg. compute a multi-band pyramid)
  return mapping->getMerger().prepareMergeAsync(Core::EQUIRECTANGULAR, *mapping, stream);
}

Status CubemapPimpl::warp(Core::ImageMapping* mapping, frameid_t frame, const Core::PanoDefinition& pano,
                          GPU::Stream& stream) {
  FAIL_RETURN(mapping->warpCubemap(frame, pano, equiangular, stream));

  // analyze the image content if needed (eg. compute a multi-band pyramid)
  FAIL_RETURN(mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_POSITIVE_X, *mapping, stream));
  FAIL_RETURN(mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_NEGATIVE_X, *mapping, stream));
  FAIL_RETURN(mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_POSITIVE_Y, *mapping, stream));
  FAIL_RETURN(mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_NEGATIVE_Y, *mapping, stream));
  FAIL_RETURN(mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_POSITIVE_Z, *mapping, stream));
  return mapping->getMerger().prepareMergeAsync(Core::CUBE_MAP_NEGATIVE_Z, *mapping, stream);
}

Status PanoPimpl::blend(const Core::PanoDefinition& pano, const Core::ImageMapping& mapping, bool firstMerger,
                        GPU::Stream& stream) {
  return mapping.getMerger().mergeAsync(Core::EQUIRECTANGULAR, pano, buffer, progressivePbo, mapping, firstMerger,
                                        stream);
}

Status CubemapPimpl::blend(const Core::PanoDefinition& pano, const Core::ImageMapping& mapping, bool firstMerger,
                           GPU::Stream& stream) {
  GPU::UniqueBuffer<uint32_t> dummy;
  FAIL_RETURN(
      mapping.getMerger().mergeAsync(Core::CUBE_MAP_POSITIVE_X, pano, buffers[0], dummy, mapping, firstMerger, stream));
  FAIL_RETURN(
      mapping.getMerger().mergeAsync(Core::CUBE_MAP_NEGATIVE_X, pano, buffers[1], dummy, mapping, firstMerger, stream));
  FAIL_RETURN(
      mapping.getMerger().mergeAsync(Core::CUBE_MAP_POSITIVE_Y, pano, buffers[2], dummy, mapping, firstMerger, stream));
  FAIL_RETURN(
      mapping.getMerger().mergeAsync(Core::CUBE_MAP_NEGATIVE_Y, pano, buffers[3], dummy, mapping, firstMerger, stream));
  FAIL_RETURN(
      mapping.getMerger().mergeAsync(Core::CUBE_MAP_POSITIVE_Z, pano, buffers[4], dummy, mapping, firstMerger, stream));
  return mapping.getMerger().mergeAsync(Core::CUBE_MAP_NEGATIVE_Z, pano, buffers[5], dummy, mapping, firstMerger,
                                        stream);
}

Status PanoPimpl::flatten() { return Status::OK(); }

Status CubemapPimpl::flatten() {
  if (layout == YOUTUBE) {
    FAIL_RETURN(memcpy2DAsync(buffer, buffers[0], 0, 0, 0, 0, length, length, length, width, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, buffers[1], 0, 0, length, 0, length, length, length, width, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, buffers[2], 0, 0, 2 * length, 0, length, length, length, width, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, buffers[3], 0, 0, 0, length, length, length, length, width, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, buffers[4], 0, 0, length, length, length, length, length, width, stream));
    return memcpy2DAsync(buffer, buffers[5], 0, 0, 2 * length, length, length, length, length, width, stream);
  } else if (layout == ROT) {
    FAIL_RETURN(Image::rotate(tmp, buffers[1], length, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, tmp, 0, 0, 0, 0, length, length, length, width, stream));
    FAIL_RETURN(Image::rotate(tmp, buffers[5], length, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, tmp, 0, 0, length, 0, length, length, length, width, stream));
    FAIL_RETURN(Image::rotate(tmp, buffers[0], length, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, tmp, 0, 0, 2 * length, 0, length, length, length, width, stream));
    FAIL_RETURN(Image::rotateLeft(tmp, buffers[2], length, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, tmp, 0, 0, 0, length, length, length, length, width, stream));
    FAIL_RETURN(Image::rotateLeft(tmp, buffers[4], length, stream));
    FAIL_RETURN(memcpy2DAsync(buffer, tmp, 0, 0, length, length, length, length, length, width, stream));
    FAIL_RETURN(Image::rotateLeft(tmp, buffers[3], length, stream));
    return memcpy2DAsync(buffer, tmp, 0, 0, 2 * length, length, length, length, length, width, stream);
  }
  assert(false);
  return Status::OK();
}

Status PanoPimpl::reconstruct(const Core::PanoDefinition& pano, const Core::ImageMapping& mapping, GPU::Stream& stream,
                              bool final) {
  if (mapping.getMerger().isMultiScale()) {
    return mapping.reconstruct(Core::EQUIRECTANGULAR, pano, progressivePbo.borrow(), final, stream);
  } else {
    return mapping.reconstruct(Core::EQUIRECTANGULAR, pano, buffer, final, stream);
  }
}

Status CubemapPimpl::reconstruct(const Core::PanoDefinition& pano, const Core::ImageMapping& mapping, GPU::Stream&,
                                 bool final) {
  GPU::Buffer<uint32_t> dummy;

  FAIL_RETURN(mapping.reconstruct(Core::CUBE_MAP_POSITIVE_X, pano, dummy, final, stream));
  FAIL_RETURN(mapping.reconstruct(Core::CUBE_MAP_NEGATIVE_X, pano, dummy, final, stream));
  FAIL_RETURN(mapping.reconstruct(Core::CUBE_MAP_POSITIVE_Y, pano, dummy, final, stream));
  FAIL_RETURN(mapping.reconstruct(Core::CUBE_MAP_NEGATIVE_Y, pano, dummy, final, stream));
  FAIL_RETURN(mapping.reconstruct(Core::CUBE_MAP_POSITIVE_Z, pano, dummy, final, stream));
  return mapping.reconstruct(Core::CUBE_MAP_NEGATIVE_Z, pano, dummy, final, stream);
}

void SourceSurface::accept(std::shared_ptr<SourceRenderer>, mtime_t) {}

void SourceOpenGLSurface::accept(std::shared_ptr<SourceRenderer> renderer, mtime_t date) {
  renderer->render(std::dynamic_pointer_cast<SourceOpenGLSurface>(shared_from_this()), date);
}

void PanoSurface::accept(std::shared_ptr<PanoRenderer>, mtime_t) {}

void PanoSurface::accept(const std::shared_ptr<GPU::Overlayer>&, std::shared_ptr<PanoOpenGLSurface>, mtime_t) {}

void PanoOpenGLSurface::accept(std::shared_ptr<PanoRenderer> renderer, mtime_t date) {
  renderer->render(std::dynamic_pointer_cast<PanoOpenGLSurface>(shared_from_this()), date);
}

void PanoOpenGLSurface::accept(const std::shared_ptr<GPU::Overlayer>& compositor,
                               std::shared_ptr<PanoOpenGLSurface> oglSurf, mtime_t date) {
  compositor->computeOverlay(std::dynamic_pointer_cast<PanoOpenGLSurface>(shared_from_this()), oglSurf, date);
}

void CubemapSurface::accept(std::shared_ptr<PanoRenderer>, mtime_t) {}

void CubemapSurface::accept(const std::shared_ptr<GPU::Overlayer>&, std::shared_ptr<PanoOpenGLSurface>, mtime_t) {}

void CubemapOpenGLSurface::accept(std::shared_ptr<PanoRenderer> renderer, mtime_t date) {
  if ((static_cast<CubemapPimpl*>(pimpl))->equiangular) {
    renderer->renderEquiangularCubemap(std::dynamic_pointer_cast<CubemapOpenGLSurface>(shared_from_this()), date);
  } else {
    renderer->renderCubemap(std::dynamic_pointer_cast<CubemapOpenGLSurface>(shared_from_this()), date);
  }
}

void CubemapOpenGLSurface::accept(const std::shared_ptr<GPU::Overlayer>&, std::shared_ptr<PanoOpenGLSurface>, mtime_t) {
}

}  // namespace Core
}  // namespace VideoStitch