readerController load audio into map<blockID, audioData>

The change is needed as MSVC doesn't allow for a vector of a map
of a non-copyable object.

We can't use a list, as apparently several calls to loadAudio need
to write the blocks of different audio groups into the same position
in the output vector. std::list doesn't have random access.

Using a map now, so we can keep the previous behavior. The int64_t key
is only used in construction, and can be disregarded by the loading
code.

lib/src/audio/audioPipeline.hpp

@@ -25,6 +25,11 @@ namespace Audio {
 typedef std::map<readerid_t, AudioBlock> audioBlockReaderMap_t;
 typedef std::map<groupid_t, audioBlockReaderMap_t> audioBlockGroupMap_t;
 
+// originally a vector<audioBlockGroupMap_t>, but this doesn't compile in MSVC
+// (https://stackoverflow.com/questions/58700780/vector-of-maps-of-non-copyable-objects-fails-to-compile-in-msvc)
+// -> using std::map, where key is audio block index and can be disregarded
+typedef std::map<int64_t, audioBlockGroupMap_t> audioBlocks_t;
+
 static const std::string kAudioPipeTag{"audiopipeline"};
 
 class AudioPipeline {

lib/src/core/controller.cpp

@@ -619,7 +619,7 @@ template <typename VideoPipeline>
 ControllerStatus ControllerImpl<VideoPipeline>::extract(ExtractOutput* extract, bool readFrame) {
   // load the acquisition data
   std::map<readerid_t, Input::PotentialFrame> inputBuffers;
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   Input::MetadataChunk metadata;
 
   mtime_t date;
@@ -644,7 +644,7 @@ ControllerStatus ControllerImpl<VideoPipeline>::extract(std::vector<ExtractOutpu
                                                         bool readFrame) {
   // load the acquisition data
   std::map<readerid_t, Input::PotentialFrame> inputBuffers;
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   Input::MetadataChunk metadata;
 
   mtime_t date;
@@ -734,7 +734,7 @@ ControllerStatus ControllerImpl<VideoPipeline>::stitchAndExtract(Output* output,
   // load the acquisition data
   std::map<readerid_t, Input::PotentialFrame> inputBuffers;
   mtime_t date;
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   Input::MetadataChunk metadata;
 
   if (readFrame) {
@@ -752,8 +752,8 @@ ControllerStatus ControllerImpl<VideoPipeline>::stitchAndExtract(Output* output,
   // process audio
   if (statusAudio.ok() || statusAudio.getCode() == Input::ReadStatusCode::TryAgain) {
     for (auto& samples : audioBlocks) {
-      if (!samples.empty()) {
-        audioPipe->process(samples);
+      if (!samples.second.empty()) {
+        audioPipe->process(samples.second);
       }
     }
   }

lib/src/core/controllerInputFrames.cpp

@@ -82,7 +82,7 @@ Status ControllerInputFrames<destinationColor, readbackType>::load(
     std::map<readerid_t, PotentialValue<GPU::HostBuffer<readbackType>>>& processedFrames, mtime_t* date) {
   std::map<readerid_t, Input::PotentialFrame> framesFromReader;
   processedFrames.clear();
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   mtime_t tempDate = 0;
   Input::MetadataChunk metadata;
   auto loadStatus = readerController->load(tempDate, framesFromReader, audioBlocks, metadata);

lib/src/core/readerController.cpp

@@ -186,7 +186,7 @@ ReaderController::~ReaderController() {
 
 std::tuple<Input::ReadStatus, Input::ReadStatus, Input::ReadStatus> ReaderController::load(
     mtime_t& date, std::map<readerid_t, Input::PotentialFrame>& frames,
-    std::list<Audio::audioBlockGroupMap_t>& audioBlocks, Input::MetadataChunk& metadata) {
+    Audio::audioBlocks_t& audioBlocks, Input::MetadataChunk& metadata) {
   // protect from concurrent seeks
   std::lock_guard<std::mutex> lock(inputMutex);
 
@@ -370,7 +370,7 @@ Input::ReadStatus ReaderController::loadVideo(mtime_t& date, std::map<readerid_t
 /// \param audioBlocks First dimension is the block, second index the input
 ///                    (e.g. audioIn[1][0] => second block of the input 0)
 /// \return Code::Ok on success, else an error code
-Input::ReadStatus ReaderController::loadAudio(std::list<Audio::audioBlockGroupMap_t>& audioBlocks, groupid_t gr) {
+Input::ReadStatus ReaderController::loadAudio(Audio::audioBlocks_t& audioBlocks, groupid_t gr) {
   std::list<std::map<readerid_t, Audio::Samples>> audioIn;
 
   size_t nbSamples = audioPipeDef->getBlockSize();
@@ -489,6 +489,7 @@ Input::ReadStatus ReaderController::loadAudio(std::list<Audio::audioBlockGroupMa
 
   Audio::AudioBlock blk;
   // First dimension is the block, second dimension the input (e.g. audioIn[1][0] => second block of the first input)
+  int64_t audioInIdx = 0;
   for (auto& samplesByReader : audioIn) {  // for each block
     Audio::audioBlockReaderMap_t audioBlockPerReader;
     for (auto& readerData : samplesByReader) {  // for each reader
@@ -503,9 +504,15 @@ Input::ReadStatus ReaderController::loadAudio(std::list<Audio::audioBlockGroupMa
       audioBlockPerReader[readerData.first] = std::move(blk);
     }
 
-    Audio::audioBlockGroupMap_t audioBlockPerGroup;
-    audioBlockPerGroup[gr] = std::move(audioBlockPerReader);
-    audioBlocks.push_back(std::move(audioBlockPerGroup));
+    if (audioBlocks.find(audioInIdx) != audioBlocks.end()) {
+      audioBlocks[audioInIdx][gr] = std::move(audioBlockPerReader);
+    } else {
+      Audio::audioBlockGroupMap_t audioBlockPerGroup;
+      audioBlockPerGroup[gr] = std::move(audioBlockPerReader);
+      audioBlocks[audioInIdx] = std::move(audioBlockPerGroup);
+    }
+
+    audioInIdx++;
   }
 
   Logger::verbose(CTRLtag) << "read group " << gr << " " << audioIn.size() << " audio blocks starting at timestamp "
@@ -814,4 +821,4 @@ Status ReaderController::setupAudioPreProc(const std::string& name, groupid_t gr
 }
 
 }  // namespace Core
-}  // namespace VideoStitch
\ No newline at end of file
+}  // namespace VideoStitch

lib/src/core/readerController.hpp

@@ -55,7 +55,7 @@ class ReaderController {
 
   std::tuple<Input::ReadStatus, Input::ReadStatus, Input::ReadStatus> load(
       mtime_t&, std::map<readerid_t, Input::PotentialFrame>& frames,
-      std::list<Audio::audioBlockGroupMap_t>& audioBlocks, Input::MetadataChunk& imu_metadata);
+      Audio::audioBlocks_t& audioBlocks, Input::MetadataChunk& imu_metadata);
   mtime_t reload(std::map<readerid_t, Input::PotentialFrame>& frames);
   void releaseBuffer(std::map<readerid_t, Input::PotentialFrame>& frames);
 
@@ -148,7 +148,7 @@ class ReaderController {
 
  private:
   Input::ReadStatus loadVideo(mtime_t& date, std::map<readerid_t, Input::PotentialFrame>& frames);
-  Input::ReadStatus loadAudio(std::list<Audio::audioBlockGroupMap_t>& audioIn, groupid_t gr);
+  Input::ReadStatus loadAudio(Audio::audioBlocks_t& audioIn, groupid_t gr);
   Input::ReadStatus loadMetadata(Input::MetadataChunk& Measure);
 
   /**

lib/src/coredepth/depthController.cpp

@@ -59,7 +59,7 @@ ControllerStatus DepthControllerImpl::estimateDepth(std::vector<ExtractOutput*>
   // load the acquisition data
   std::map<readerid_t, Input::PotentialFrame> inputBuffers;
   mtime_t date;
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   Input::MetadataChunk metadata;
 
   std::tie(statusVideo, statusAudio, statusMetadata) =

lib/src/test/readerControllerTest.cpp

@@ -117,7 +117,7 @@ void testLoadedDate() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   readerController->load(date, frames, audio, metadata);
@@ -125,7 +125,7 @@ void testLoadedDate() {
   ENSURE_EQ(date, (mtime_t)5 * 1000 * 1000, "ReaderController should take timing information from audio reader");
   ENSURE_EQ(audio.empty(), false, "audio should not be empty");
   for (auto &audioGr : audio) {
-    for (auto &blockmap : audioGr) {
+    for (auto &blockmap : audioGr.second) {
       ENSURE_EQ((int)blockmap.second.size(), 2, "Should return a block with two inputs");
       ENSURE_EQ((int)blockmap.second.begin()->first, 3, "Check first input id audio");
       ENSURE_EQ((int)blockmap.second.rbegin()->first, 4, "Check second input id audio");
@@ -198,7 +198,7 @@ void testLoadedDateWithAudioReader() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   readerController->load(date, frames, audio, metadata);
@@ -240,7 +240,7 @@ void testSeekWithImage() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   readerController->load(date, frames, audio, metadata);
@@ -294,7 +294,7 @@ void testEOSAudio() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   std::tuple<Input::ReadStatus, Input::ReadStatus, Input::ReadStatus> status =
@@ -326,7 +326,7 @@ void testTryAgainAudio() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   std::tuple<Input::ReadStatus, Input::ReadStatus, Input::ReadStatus> status =
@@ -357,7 +357,7 @@ void testAudioVideoResync() {
 
   std::map<readerid_t, Input::PotentialFrame> frames;
   Input::MetadataChunk metadata;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
 
   std::tuple<Input::ReadStatus, Input::ReadStatus, Input::ReadStatus> status =
@@ -370,7 +370,7 @@ void testAudioVideoResync() {
   auto sr = Audio::getDefaultSamplingRate();
   mtime_t blockDuration = (mtime_t)std::round((blockSize * 1000000. / sr));
   for (auto &grMap : audio) {
-    for (auto &readerMap : grMap) {
+    for (auto &readerMap : grMap.second) {
       for (auto &kv : readerMap.second) {
         mtime_t tmp = (blockDuration * iBlk);
         ENSURE_EQ((mtime_t)(date + tmp), kv.second.getTimestamp(), "test audio video synchro");
@@ -378,7 +378,7 @@ void testAudioVideoResync() {
     }
     iBlk++;
   }
-  ENSURE_EQ(date, audio.front()[0][0].getTimestamp(), "audio and video should be synchro");
+  ENSURE_EQ(date, audio[0][0][0].getTimestamp(), "audio and video should be synchro");
   readerController->releaseBuffer(frames);
 }
 
@@ -421,7 +421,7 @@ void testInputGroups(int fps, ReaderClockResolution clockResolution) {
     }
 
     std::map<readerid_t, Input::PotentialFrame> frames;
-    std::list<Audio::audioBlockGroupMap_t> audio;
+    Audio::audioBlocks_t audio;
     mtime_t date;
     Input::MetadataChunk metadata;
 
@@ -536,7 +536,7 @@ void testCurrentFrame() {
   ENSURE(readerController.status());
 
   std::map<readerid_t, Input::PotentialFrame> frames;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t date;
   Input::MetadataChunk metadata;
 
@@ -660,7 +660,7 @@ void testAudioVideoSynchro() {
   auto readerController = Core::ReaderController::create(*panoDef, *audioPipe, readerFactory);
   ENSURE(readerController.status());
   std::map<readerid_t, Input::PotentialFrame> frames;
-  std::list<Audio::audioBlockGroupMap_t> audio;
+  Audio::audioBlocks_t audio;
   mtime_t videoDate;
   Input::MetadataChunk metadata;
 
@@ -675,15 +675,16 @@ void testAudioVideoSynchro() {
                    1.5);
       ENSURE_EQ(nbAudioBlocksExpected, audio.size(), "Check number of audio blocks at the first load");
       // Check that the first audio frame of the audio video group are well synchronized
-      ENSURE_EQ(videoDate, audio.front().at(audioVideoGrId).at(0).getTimestamp(),
+      ENSURE_EQ(videoDate, audio.at(0).at(audioVideoGrId).at(0).getTimestamp(),
                 "Check audio video synchronization of the audio video group");
-      ENSURE_EQ(videoDate, audio.front().at(audioVideoGrId).at(1).getTimestamp(),
+      ENSURE_EQ(videoDate, audio.at(0).at(audioVideoGrId).at(1).getTimestamp(),
                 "Check audio video synchronization of the audio video group");
-      ENSURE_EQ(videoDate, audio.front().at(audioOnlyGrId).at(2).getTimestamp(),
+      ENSURE_EQ(videoDate, audio.at(0).at(audioOnlyGrId).at(2).getTimestamp(),
                 "Check audio video synchronization of the audio video group");
     }
 
-    for (const Audio::audioBlockGroupMap_t &audioPerGroup : audio) {
+    for (const auto& pair : audio) {
+      const Audio::audioBlockGroupMap_t &audioPerGroup = pair.second;
       ENSURE_EQ((size_t)2, audioPerGroup.size(), "Check number of groups loaded");
       mtime_t timestampAudioVideo = -1;
       mtime_t timestampAudioOnly = -1;

lib/src/undistort/undistortController.cpp

@@ -59,7 +59,7 @@ ControllerStatus UndistortControllerImpl::undistort(std::vector<ExtractOutput*>
   // load the acquisition data
   std::map<readerid_t, Input::PotentialFrame> inputBuffers;
   mtime_t date;
-  std::list<Audio::audioBlockGroupMap_t> audioBlocks;
+  Audio::audioBlocks_t audioBlocks;
   Input::MetadataChunk metadata;
 
   std::tie(statusVideo, statusAudio, statusMetadata) =