Added feedback from packetization to picturemanager to address a dead…

…lock. Deadlock was due to base layer pictures waiting for the previous one. Once testing is done, I will removed the macros
OpenVisualCloud · lijing0010 · Jul 22, 2019 · Dec 11, 2018 · Dec 18, 2018 · Jan 8, 2019
commit f1cd4c53e18fd25ffad9b5d8a5e871e70cb38e18
diff --git a/Source/Lib/Codec/EbDefinitions.h b/Source/Lib/Codec/EbDefinitions.h
@@ -10,7 +10,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#define PACK_FEEDBACK 1
 // Internal Marcos
 #define NON_AVX512_SUPPORT
 

diff --git a/Source/Lib/Codec/EbEncHandle.c b/Source/Lib/Codec/EbEncHandle.c
@@ -1127,7 +1127,11 @@ EB_API EB_ERRORTYPE EbInitEncoder(EB_COMPONENTTYPE *h265EncComponent)
         return_error = EbSystemResourceCtor(
             &encHandlePtr->pictureDemuxResultsResourcePtr,
             encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->pictureDemuxFifoInitCount,
+#if PACK_FEEDBACK
+            encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->sourceBasedOperationsProcessInitCount + encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->encDecProcessInitCount + 1, // 1 for packetization
+#else
 			encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->sourceBasedOperationsProcessInitCount + encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->encDecProcessInitCount,
+#endif     
             EB_PictureManagerProcessInitCount,
             &encHandlePtr->pictureDemuxResultsProducerFifoPtrArray,
             &encHandlePtr->pictureDemuxResultsConsumerFifoPtrArray,
@@ -1446,7 +1450,11 @@ EB_API EB_ERRORTYPE EbInitEncoder(EB_COMPONENTTYPE *h265EncComponent)
     return_error = PacketizationContextCtor(
         (PacketizationContext_t**) &encHandlePtr->packetizationContextPtr,
         encHandlePtr->entropyCodingResultsConsumerFifoPtrArray[0],
-        encHandlePtr->rateControlTasksProducerFifoPtrArray[RateControlPortLookup(RATE_CONTROL_INPUT_PORT_PACKETIZATION, 0)]);
+        encHandlePtr->rateControlTasksProducerFifoPtrArray[RateControlPortLookup(RATE_CONTROL_INPUT_PORT_PACKETIZATION, 0)]
+#if PACK_FEEDBACK
+        ,encHandlePtr->pictureDemuxResultsProducerFifoPtrArray[encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->sourceBasedOperationsProcessInitCount + encHandlePtr->sequenceControlSetInstanceArray[0]->sequenceControlSetPtr->encDecProcessInitCount] // Add port lookup logic here JMJ
+#endif
+    );
 
     if (return_error == EB_ErrorInsufficientResources){
         return EB_ErrorInsufficientResources;

diff --git a/Source/Lib/Codec/EbPacketizationProcess.c b/Source/Lib/Codec/EbPacketizationProcess.c
@@ -17,6 +17,9 @@
 #include "EbRateControlTasks.h"
 #include "EbRateControlProcess.h"
 #include "EbTime.h"
+#if PACK_FEEDBACK
+#include "EbPictureDemuxResults.h"
+#endif
 
 void HrdFullness(SequenceControlSet_t *sequenceControlSetPtr, PictureControlSet_t *pictureControlSetptr, AppBufferingPeriodSei_t *seiBP)
 {
@@ -95,14 +98,21 @@ void InitHRD(SequenceControlSet_t *scsPtr)
 EB_ERRORTYPE PacketizationContextCtor(
     PacketizationContext_t **contextDblPtr,
     EbFifo_t                *entropyCodingInputFifoPtr,
-    EbFifo_t                *rateControlTasksOutputFifoPtr)
+    EbFifo_t                *rateControlTasksOutputFifoPtr
+#if PACK_FEEDBACK
+    ,EbFifo_t                *pictureManagerOutputFifoPtr
+#endif
+)
 {
     PacketizationContext_t *contextPtr;
     EB_MALLOC(PacketizationContext_t*, contextPtr, sizeof(PacketizationContext_t), EB_N_PTR);
     *contextDblPtr = contextPtr;
 
     contextPtr->entropyCodingInputFifoPtr      = entropyCodingInputFifoPtr;
     contextPtr->rateControlTasksOutputFifoPtr  = rateControlTasksOutputFifoPtr;
+#if PACK_FEEDBACK
+    contextPtr->pictureManagerOutputFifoPtr    = pictureManagerOutputFifoPtr;
+#endif
 
     EB_MALLOC(EbPPSConfig_t*, contextPtr->ppsConfig, sizeof(EbPPSConfig_t), EB_N_PTR);
 
@@ -131,7 +141,10 @@ void* PacketizationKernel(void *inputPtr)
     EB_BUFFERHEADERTYPE           *outputStreamPtr;
     EbObjectWrapper_t              *rateControlTasksWrapperPtr;
     RateControlTasks_t             *rateControlTasksPtr;
-
+#if PACK_FEEDBACK
+    EbObjectWrapper_t               *pictureManagerResultsWrapperPtr;
+    PictureDemuxResults_t       	*pictureManagerResultPtr;
+#endif
     // Bitstream copy to output buffer
     Bitstream_t                     bitstream;
 
@@ -197,7 +210,25 @@ void* PacketizationKernel(void *inputPtr)
         rateControlTasksPtr                                 = (RateControlTasks_t*) rateControlTasksWrapperPtr->objectPtr; 
         rateControlTasksPtr->pictureControlSetWrapperPtr    = pictureControlSetPtr->PictureParentControlSetWrapperPtr;
         rateControlTasksPtr->taskType                       = RC_PACKETIZATION_FEEDBACK_RESULT;
-
+
+#if PACK_FEEDBACK
+        if (sequenceControlSetPtr->staticConfig.rateControlMode) {
+            // Get Empty Results Object
+            EbGetEmptyObject(
+                contextPtr->pictureManagerOutputFifoPtr,
+                &pictureManagerResultsWrapperPtr);
+
+            pictureManagerResultPtr = (PictureDemuxResults_t*)pictureManagerResultsWrapperPtr->objectPtr;
+            pictureManagerResultPtr->pictureNumber = pictureControlSetPtr->pictureNumber;
+            pictureManagerResultPtr->pictureType = EB_PIC_FEEDBACK;
+            pictureManagerResultPtr->sequenceControlSetWrapperPtr = pictureControlSetPtr->sequenceControlSetWrapperPtr;
+        }
+        else {
+            pictureManagerResultsWrapperPtr = EB_NULL;
+            (void) pictureManagerResultPtr;
+            (void)pictureManagerResultsWrapperPtr;
+        }
+#endif        
         sliceType = pictureControlSetPtr->sliceType;
 
         if(pictureControlSetPtr->pictureNumber == 0 && sequenceControlSetPtr->staticConfig.codeVpsSpsPps == 1) {
@@ -718,6 +749,12 @@ void* PacketizationKernel(void *inputPtr)
         // Post Rate Control Taks            
         EbPostFullObject(rateControlTasksWrapperPtr);    
 
+#if PACK_FEEDBACK
+        if (sequenceControlSetPtr->staticConfig.rateControlMode) {
+            // Post the Full Results Object
+            EbPostFullObject(pictureManagerResultsWrapperPtr);
+        }
+#endif
         //Release the Parent PCS then the Child PCS
         EbReleaseObject(entropyCodingResultsPtr->pictureControlSetWrapperPtr);//Child
 

diff --git a/Source/Lib/Codec/EbPacketizationProcess.h b/Source/Lib/Codec/EbPacketizationProcess.h
@@ -30,6 +30,9 @@ typedef struct PacketizationContext_s
     EbFifo_t                *entropyCodingInputFifoPtr;
     EbFifo_t                *rateControlTasksOutputFifoPtr;
     EbPPSConfig_t           *ppsConfig;
+#if PACK_FEEDBACK
+    EbFifo_t                *pictureManagerOutputFifoPtr;   // to picture-manager
+#endif
 
 } PacketizationContext_t;
 
@@ -39,7 +42,11 @@ typedef struct PacketizationContext_s
 extern EB_ERRORTYPE PacketizationContextCtor(
     PacketizationContext_t **contextDblPtr,
     EbFifo_t                *entropyCodingInputFifoPtr,
-    EbFifo_t                *rateControlTasksOutputFifoPtr);
+    EbFifo_t                *rateControlTasksOutputFifoPtr
+#if PACK_FEEDBACK
+    , EbFifo_t                *pictureManagerOutputFifoPtr
+#endif
+);
 
 
 extern void* PacketizationKernel(void *inputPtr);

diff --git a/Source/Lib/Codec/EbPictureDemuxResults.h b/Source/Lib/Codec/EbPictureDemuxResults.h
@@ -15,6 +15,9 @@ typedef enum EB_PIC_TYPE {
     EB_PIC_INVALID = 0,
     EB_PIC_INPUT = 1,
     EB_PIC_REFERENCE = 2
+#if PACK_FEEDBACK
+    ,EB_PIC_FEEDBACK = 3
+#endif
 } EB_PIC_TYPE;
 
 /**************************************

diff --git a/Source/Lib/Codec/EbPictureManagerProcess.c b/Source/Lib/Codec/EbPictureManagerProcess.c
@@ -526,7 +526,35 @@ void* PictureManagerKernel(void *inputPtr)
             EbReleaseObject(inputPictureDemuxPtr->sequenceControlSetWrapperPtr);
 
             break;
-
+#if PACK_FEEDBACK
+        case EB_PIC_FEEDBACK:
+
+            sequenceControlSetPtr = (SequenceControlSet_t*)inputPictureDemuxPtr->sequenceControlSetWrapperPtr->objectPtr;
+            encodeContextPtr = sequenceControlSetPtr->encodeContextPtr;
+
+            referenceQueueIndex = encodeContextPtr->referencePictureQueueHeadIndex;
+            // Find the Reference in the Reference Queue
+            do {
+
+                referenceEntryPtr = encodeContextPtr->referencePictureQueue[referenceQueueIndex];
+
+                if (referenceEntryPtr->pictureNumber == inputPictureDemuxPtr->pictureNumber) {
+
+                    // Set the feedback arrived
+                    referenceEntryPtr->feedbackArrived = EB_TRUE;
+                }
+
+                // Increment the referenceQueueIndex Iterator
+                referenceQueueIndex = (referenceQueueIndex == REFERENCE_QUEUE_MAX_DEPTH - 1) ? 0 : referenceQueueIndex + 1;
+
+            } while ((referenceQueueIndex != encodeContextPtr->referencePictureQueueTailIndex) && (referenceEntryPtr->pictureNumber != inputPictureDemuxPtr->pictureNumber));
+
+            //keep the relase of SCS here because we still need the encodeContext strucutre here
+            // Release the Reference's SequenceControlSet    
+            EbReleaseObject(inputPictureDemuxPtr->sequenceControlSetWrapperPtr);
+
+            break;
+#endif
         default:
 
             sequenceControlSetPtr   = (SequenceControlSet_t*) inputPictureDemuxPtr->sequenceControlSetWrapperPtr->objectPtr;
@@ -585,7 +613,7 @@ void* PictureManagerKernel(void *inputPtr)
                     availabilityFlag =
                         (availabilityFlag == EB_FALSE)          ? EB_FALSE  :   // Don't update if already False 
                         (refPoc > currentInputPoc)              ? EB_FALSE  :   // The Reference has not been received as an Input Picture yet, then its availability is false
-                        (!encodeContextPtr->terminatingSequenceFlagReceived && (sequenceControlSetPtr->staticConfig.rateControlMode && entryPictureControlSetPtr->sliceType != EB_I_PICTURE && entryPictureControlSetPtr->temporalLayerIndex == 0 && !referenceEntryPtr->feedbackArrived)) ? EB_FALSE :
+                        (!encodeContextPtr->terminatingSequenceFlagReceived && (entrySequenceControlSetPtr->staticConfig.rateControlMode && entryPictureControlSetPtr->sliceType != EB_I_PICTURE && entryPictureControlSetPtr->temporalLayerIndex == 0 && !referenceEntryPtr->feedbackArrived)) ? EB_FALSE :
                         (referenceEntryPtr->referenceAvailable) ? EB_TRUE   :   // The Reference has been completed
                                                                   EB_FALSE;     // The Reference has not been completed
                 }
@@ -622,7 +650,7 @@ void* PictureManagerKernel(void *inputPtr)
                             availabilityFlag =
                                 (availabilityFlag == EB_FALSE)          ? EB_FALSE  :   // Don't update if already False 
                                 (refPoc > currentInputPoc)              ? EB_FALSE  :   // The Reference has not been received as an Input Picture yet, then its availability is false
-                                (!encodeContextPtr->terminatingSequenceFlagReceived && (sequenceControlSetPtr->staticConfig.rateControlMode && entryPictureControlSetPtr->sliceType != EB_I_PICTURE && entryPictureControlSetPtr->temporalLayerIndex == 0 && !referenceEntryPtr->feedbackArrived)) ? EB_FALSE :
+                                (!encodeContextPtr->terminatingSequenceFlagReceived && (entrySequenceControlSetPtr->staticConfig.rateControlMode && entryPictureControlSetPtr->sliceType != EB_I_PICTURE && entryPictureControlSetPtr->temporalLayerIndex == 0 && !referenceEntryPtr->feedbackArrived)) ? EB_FALSE :
                                 (referenceEntryPtr->referenceAvailable) ? EB_TRUE   :   // The Reference has been completed
                                                                           EB_FALSE;     // The Reference has not been completed
                         }
@@ -718,7 +746,7 @@ void* PictureManagerKernel(void *inputPtr)
 					ChildPictureControlSetPtr->useDeltaQp = (EB_U8)(entrySequenceControlSetPtr->staticConfig.improveSharpness || entrySequenceControlSetPtr->staticConfig.bitRateReduction);
 
                     // Check resolution
-                    if (sequenceControlSetPtr->inputResolution < INPUT_SIZE_1080p_RANGE)
+                    if (entrySequenceControlSetPtr->inputResolution < INPUT_SIZE_1080p_RANGE)
                         ChildPictureControlSetPtr->difCuDeltaQpDepth = 2;
                     else
                         ChildPictureControlSetPtr->difCuDeltaQpDepth = 3;

diff --git a/Source/Lib/Codec/EbRateControlProcess.c b/Source/Lib/Codec/EbRateControlProcess.c
@@ -2676,7 +2676,7 @@ void* RateControlKernel(void *inputPtr)
 
             // Update feedback arrived in referencepictureQueue
            // printf("RC FEEDBACK ARRIVED %d\n", (int)parentPictureControlSetPtr->pictureNumber);
-
+#if !PACK_FEEDBACK
             ReferenceQueueEntry_t           *referenceEntryPtr;
             EB_U32                          referenceQueueIndex;
             encodeContextPtr = sequenceControlSetPtr->encodeContextPtr;
@@ -2696,7 +2696,7 @@ void* RateControlKernel(void *inputPtr)
                 referenceQueueIndex = (referenceQueueIndex == REFERENCE_QUEUE_MAX_DEPTH - 1) ? 0 : referenceQueueIndex + 1;
 
             } while ((referenceQueueIndex != encodeContextPtr->referencePictureQueueTailIndex) && (referenceEntryPtr->pictureNumber != parentPictureControlSetPtr->pictureNumber));
-
+#endif
             // Frame level RC
             if (sequenceControlSetPtr->intraPeriodLength == -1 || sequenceControlSetPtr->staticConfig.rateControlMode == 0){
                 rateControlParamPtr = contextPtr->rateControlParamQueue[0];

diff --git a/Source/Lib/Codec/EbResourceCoordinationProcess.c b/Source/Lib/Codec/EbResourceCoordinationProcess.c
@@ -588,13 +588,29 @@ void* ResourceCoordinationKernel(void *inputPtr)
 
         }
         EbReleaseMutex(contextPtr->sequenceControlSetInstanceArray[instanceIndex]->configMutex);
+#if PACK_FEEDBACK
+        if (sequenceControlSetPtr->staticConfig.rateControlMode) {
+            // Sequence Control Set is released by Rate Control after passing through MDC->MD->ENCDEC->Packetization->RateControl
+            //   ,in the PictureManager after receiving the reference and in PictureManager after receiving the feedback
+            EbObjectIncLiveCount(
+                contextPtr->sequenceControlSetActiveArray[instanceIndex],
+                3);
+        }
+        else {
+            // Sequence Control Set is released by Rate Control after passing through MDC->MD->ENCDEC->Packetization->RateControl
+            //   and in the PictureManager
+            EbObjectIncLiveCount(
+                contextPtr->sequenceControlSetActiveArray[instanceIndex],
+                2);
 
+        }
+#else
         // Sequence Control Set is released by Rate Control after passing through MDC->MD->ENCDEC->Packetization->RateControl
         //   and in the PictureManager
         EbObjectIncLiveCount(
             contextPtr->sequenceControlSetActiveArray[instanceIndex],
             2);
-
+#endif
         // Set the current SequenceControlSet
         sequenceControlSetPtr   = (SequenceControlSet_t*) contextPtr->sequenceControlSetActiveArray[instanceIndex]->objectPtr;
 

diff --git a/Source/Lib/Codec/EbSequenceControlSet.c b/Source/Lib/Codec/EbSequenceControlSet.c
@@ -113,7 +113,6 @@ EB_ERRORTYPE EbSequenceControlSetCtor(
     sequenceControlSetPtr->enableStrongIntraSmoothing                       = EB_TRUE;
 
     // Rate Control
-    sequenceControlSetPtr->rateControlMode                                  = 0; 
     sequenceControlSetPtr->targetBitrate                                    = 0x1000; 
     sequenceControlSetPtr->availableBandwidth                               = 0x1000;
 

diff --git a/Source/Lib/Codec/EbSequenceControlSet.h b/Source/Lib/Codec/EbSequenceControlSet.h
@@ -90,7 +90,6 @@ typedef struct SequenceControlSet_s
     EB_U32                      generalFrameOnlyConstraintFlag;
 
     // Rate Control
-    EB_U32                      rateControlMode;
     EB_U32                      targetBitrate;
     EB_U32                      availableBandwidth;