GetInterMerge Candidates () function is a very important function in inter-frame prediction technology. It is the constructor of Merge list. All Merge modes must first construct a Merge candidate list, and then extend the corresponding Merge list on the basis of this regular Merge list according to different modes. For example, the Merge list of MMVD needs to be based on the regular Merge list. Go up and select two initial MVs, then construct MMVDMerge.
Ordinary Merge lists are structured roughly as follows:
We can construct a list of candidates according to the following figure:
First, check whether the adjacent blocks in the spatial domain are available MV. The order of inspection is A1 - > B1 - > B0 - > A0 - > B2.
(2) Re-examine adjacent blocks in time domain, and the order of inspection is as follows:BR->CT
(3) If the space-time domain is not filled up after the completion of the space-time domain inspection, the historical-based reference space candidate technology will be adopted, which I will follow later.
(4) Spatial average candidates
(5) If the above methods fail to fill the candidate list, the zero vector (0, 0) is used to fill the candidate list.
The original link is as follows:
H.266/VVC related technology learning notes: the basic principles of inter prediction technology in video coding
The specific code in the VTM6.0 version is as follows: I have detailed comments on the whole list construction process, and if there is anything wrong with it, you are welcome to correct it.
//The entry of candidate list functions for inter frame regular Merge is constructed, and the candidate lists of the remaining Merge modes are based on the regular Merge list to construct or directly use the regular Merge list.
void PU::getInterMergeCandidates( const PredictionUnit &pu, MergeCtx& mrgCtx,
int mmvdList,
const int& mrgCandIdx )
{
const CodingStructure &cs = *pu.cs;
const Slice &slice = *pu.cs->slice;
const uint32_t maxNumMergeCand = slice.getMaxNumMergeCand();//Get the length of the Merge list (that is, the maximum number of candidates)
const bool canFastExit = pu.cs->pps->getLog2ParallelMergeLevelMinus2() == 0;//Candidate Quick Exit
#if !JVET_L0090_PAIR_AVG
// this variable is unused if remove HEVC combined candidates
bool isCandInter[MRG_MAX_NUM_CANDS];
#endif
//Here, each candidate MV in the candidate list is initialized.
for (uint32_t ui = 0; ui < maxNumMergeCand; ++ui)
{
#if !JVET_L0090_PAIR_AVG
isCandInter[ui] = false;
#endif
mrgCtx.GBiIdx[ui] = GBI_DEFAULT;//This parameter is an index of two-way prediction weights. Each candidate has five weights to select.
mrgCtx.interDirNeighbours[ui] = 0;//Inter-frame prediction directions for each candidate adjacent block: forward, backward and bidirectional
mrgCtx.mrgTypeNeighbours [ui] = MRG_TYPE_DEFAULT_N;//Merge types of adjacent blocks for each candidate MV
//There are 12 mvField Neighbours in each Merge list, and a total of 12 candidate MVs are stored. Each candidate adjacent block can provide a forward MV and a backward MV, which can be adaptively selected to use one-way or two-way prediction.
mrgCtx.mvFieldNeighbours[(ui << 1) ].refIdx = NOT_VALID;//The forward reference frame index of each candidate's adjacent block area identifies which forward reference frame of the adjacent block is.
mrgCtx.mvFieldNeighbours[(ui << 1) + 1].refIdx = NOT_VALID;//A backward reference frame index for each candidate's adjacent block area identifying which backward reference frame of the adjacent block is.
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[ui] = false;//Whether MV uses 1/2 pixel accuracy
#endif
}
mrgCtx.numValidMergeCand = maxNumMergeCand;//Number of Maximum Available Candidates MV
// compute the location of the current PU at the center of the current PU
int cnt = 0;//Candidate counters, note the candidates that have been listed in the candidate list
const Position posLT = pu.Y().topLeft();//Upper-left adjacent position of current block space
const Position posRT = pu.Y().topRight();//Right Upper Adjacent Location of Current Block Space
const Position posLB = pu.Y().bottomLeft();//Neighboring Left-Lower Positions in Current Block Space
MotionInfo miAbove, miLeft, miAboveLeft, miAboveRight, miBelowLeft;//Motion information defining adjacent positions in five spatial domains
//First traverse the adjacent blocks on the left (A1) side of the left
const PredictionUnit* puLeft = cs.getPURestricted( posLB.offset( -1, 0 ), pu, pu.chType );//Get left block
const bool isAvailableA1 = puLeft && isDiffMER( pu, *puLeft ) && pu.cu != puLeft->cu && CU::isInter( *puLeft->cu );//Indicate whether block A1 is available
//If the left block is available
if( isAvailableA1 )
{
miLeft = puLeft->getMotionInfo( posLB.offset(-1, 0) );//Get the motion information of the left block
#if !JVET_L0090_PAIR_AVG
isCandInter[cnt] = true;
#endif
// Obtaining Interframe Prediction Direction
mrgCtx.interDirNeighbours[cnt] = miLeft.interDir;//The prediction direction of candidate adjacent blocks is obtained to the current coding block.
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = miLeft.useAltHpelIf;//Using 1/2 Pixel Accuracy
#endif
mrgCtx.GBiIdx[cnt] = (mrgCtx.interDirNeighbours[cnt] == 3) ? puLeft->cu->GBiIdx : GBI_DEFAULT;//If it is bidirectional prediction, the weight coefficients are selected adaptively, otherwise the default idx is 2.
// Getting Forward MV from Left Block
mrgCtx.mvFieldNeighbours[cnt << 1].setMvField(miLeft.mv[0], miLeft.refIdx[0]);
if (slice.isInterB())//If it is a B frame, retrieve the backward MV
{
mrgCtx.mvFieldNeighbours[(cnt << 1) + 1].setMvField(miLeft.mv[1], miLeft.refIdx[1]);
}
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;
}
//Early termination
if (cnt == maxNumMergeCand)//If the current total number of candidates equals the maximum number of Merge candidates
{
return;
}
//Traversing the adjacent blocks above above (B1), the following process is the same as the processing of the left block, you can refer to the processing of the left block.
const PredictionUnit *puAbove = cs.getPURestricted( posRT.offset( 0, -1 ), pu, pu.chType );
bool isAvailableB1 = puAbove && isDiffMER( pu, *puAbove ) && pu.cu != puAbove->cu && CU::isInter( *puAbove->cu );
if( isAvailableB1 )
{
miAbove = puAbove->getMotionInfo( posRT.offset( 0, -1 ) );
if( !isAvailableA1 || ( miAbove != miLeft ) )
{
#if !JVET_L0090_PAIR_AVG
isCandInter[cnt] = true;
#endif
// get Inter Dir
mrgCtx.interDirNeighbours[cnt] = miAbove.interDir;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = miAbove.useAltHpelIf;
#endif
// get Mv from Above
mrgCtx.GBiIdx[cnt] = (mrgCtx.interDirNeighbours[cnt] == 3) ? puAbove->cu->GBiIdx : GBI_DEFAULT;
mrgCtx.mvFieldNeighbours[cnt << 1].setMvField( miAbove.mv[0], miAbove.refIdx[0] );
if( slice.isInterB() )
{
mrgCtx.mvFieldNeighbours[( cnt << 1 ) + 1].setMvField( miAbove.mv[1], miAbove.refIdx[1] );
}
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;
}
}
// early termination
if( cnt == maxNumMergeCand )
{
return;
}
int spatialCandPos = cnt;
// above right
//Traversing through the upper right (B0) adjacent blocks, the following process is the same as the processing of the left block, you can refer to the processing of the left block.
const PredictionUnit *puAboveRight = cs.getPURestricted( posRT.offset( 1, -1 ), pu, pu.chType );
bool isAvailableB0 = puAboveRight && isDiffMER( pu, *puAboveRight ) && CU::isInter( *puAboveRight->cu );
if( isAvailableB0 )
{
miAboveRight = puAboveRight->getMotionInfo( posRT.offset( 1, -1 ) );
if( !isAvailableB1 || ( miAbove != miAboveRight ) )
{
#if !JVET_L0090_PAIR_AVG
isCandInter[cnt] = true;
#endif
// get Inter Dir
mrgCtx.interDirNeighbours[cnt] = miAboveRight.interDir;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = miAboveRight.useAltHpelIf;
#endif
// get Mv from Above-right
mrgCtx.GBiIdx[cnt] = (mrgCtx.interDirNeighbours[cnt] == 3) ? puAboveRight->cu->GBiIdx : GBI_DEFAULT;
mrgCtx.mvFieldNeighbours[cnt << 1].setMvField( miAboveRight.mv[0], miAboveRight.refIdx[0] );
if( slice.isInterB() )
{
mrgCtx.mvFieldNeighbours[( cnt << 1 ) + 1].setMvField( miAboveRight.mv[1], miAboveRight.refIdx[1] );
}
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;
}
}
// early termination
if( cnt == maxNumMergeCand )
{
return;
}
//left bottom
//Traversing through the lower left (A0) adjacent blocks, the following process is the same as the processing of the left block, you can refer to the processing of the left block.
const PredictionUnit *puLeftBottom = cs.getPURestricted( posLB.offset( -1, 1 ), pu, pu.chType );
bool isAvailableA0 = puLeftBottom && isDiffMER( pu, *puLeftBottom ) && CU::isInter( *puLeftBottom->cu );
if( isAvailableA0 )
{
miBelowLeft = puLeftBottom->getMotionInfo( posLB.offset( -1, 1 ) );
if( !isAvailableA1 || ( miBelowLeft != miLeft ) )
{
#if !JVET_L0090_PAIR_AVG
isCandInter[cnt] = true;
#endif
// get Inter Dir
mrgCtx.interDirNeighbours[cnt] = miBelowLeft.interDir;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = miBelowLeft.useAltHpelIf;
#endif
mrgCtx.GBiIdx[cnt] = (mrgCtx.interDirNeighbours[cnt] == 3) ? puLeftBottom->cu->GBiIdx : GBI_DEFAULT;
// get Mv from Bottom-Left
mrgCtx.mvFieldNeighbours[cnt << 1].setMvField( miBelowLeft.mv[0], miBelowLeft.refIdx[0] );
if( slice.isInterB() )
{
mrgCtx.mvFieldNeighbours[( cnt << 1 ) + 1].setMvField( miBelowLeft.mv[1], miBelowLeft.refIdx[1] );
}
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;
}
}
// early termination
if( cnt == maxNumMergeCand )
{
return;
}
// above left
//Traversing the upper left (B2) adjacent blocks, the following process is the same as the processing of the left block, you can refer to the processing of the left block.
if ( cnt < 4 )
{
const PredictionUnit *puAboveLeft = cs.getPURestricted( posLT.offset( -1, -1 ), pu, pu.chType );
bool isAvailableB2 = puAboveLeft && isDiffMER( pu, *puAboveLeft ) && CU::isInter( *puAboveLeft->cu );
if( isAvailableB2 )
{
miAboveLeft = puAboveLeft->getMotionInfo( posLT.offset( -1, -1 ) );
if( ( !isAvailableA1 || ( miLeft != miAboveLeft ) ) && ( !isAvailableB1 || ( miAbove != miAboveLeft ) ) )
{
#if !JVET_L0090_PAIR_AVG
isCandInter[cnt] = true;
#endif
// get Inter Dir
mrgCtx.interDirNeighbours[cnt] = miAboveLeft.interDir;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = miAboveLeft.useAltHpelIf;
#endif
mrgCtx.GBiIdx[cnt] = (mrgCtx.interDirNeighbours[cnt] == 3) ? puAboveLeft->cu->GBiIdx : GBI_DEFAULT;
// get Mv from Above-Left
mrgCtx.mvFieldNeighbours[cnt << 1].setMvField( miAboveLeft.mv[0], miAboveLeft.refIdx[0] );
if( slice.isInterB() )
{
mrgCtx.mvFieldNeighbours[( cnt << 1 ) + 1].setMvField( miAboveLeft.mv[1], miAboveLeft.refIdx[1] );
}
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;
}
}
}
// early termination
if (cnt == maxNumMergeCand)
{
return;
}
//Here, the traversal of adjacent blocks in time domain is started, and the motion information of the lower right (RB) and central (CT) positions of reference blocks in time domain is traversed.
if (slice.getEnableTMVPFlag() && (pu.lumaSize().width + pu.lumaSize().height > 12))
{
//> MTK colocated-RightBottom lower right parity block
// offset the pos to be sure to "point" to the same position the uiAbsPartIdx would've pointed to
Position posRB = pu.Y().bottomRight().offset( -3, -3 );//Location of the right-lower-corner parity block: Make an appropriate offset at the position of the right-lower corner of the current brightness PU
const PreCalcValues& pcv = *cs.pcv;
Position posC0;
Position posC1 = pu.Y().center();//The central parity block is the central position of the current brightness PU
bool C0Avail = false;
if (((posRB.x + pcv.minCUWidth) < pcv.lumaWidth) && ((posRB.y + pcv.minCUHeight) < pcv.lumaHeight))
{
int posYInCtu = posRB.y & pcv.maxCUHeightMask;
if (posYInCtu + 4 < pcv.maxCUHeight)
{
posC0 = posRB.offset(4, 4);
C0Avail = true;
}
}
Mv cColMv;//MV of the current PU isotope block
int iRefIdx = 0;//Reference Frame Index
int dir = 0;//Prediction direction
unsigned uiArrayAddr = cnt;//Location of Time Domain Candidates in Candidate Queues
bool bExistMV = ( C0Avail && getColocatedMVP(pu, REF_PIC_LIST_0, posC0, cColMv, iRefIdx ) )
|| getColocatedMVP( pu, REF_PIC_LIST_0, posC1, cColMv, iRefIdx );
//This function adjusts the scaling ratio of MV in time domain, and finally gets the motion information of the scaling PU.
if (bExistMV)
{
dir |= 1;//1-bit or
mrgCtx.mvFieldNeighbours[2 * uiArrayAddr].setMvField(cColMv, iRefIdx);//Getting Forward MV of Colocation Block
}
if (slice.isInterB())//If it is a B frame, the backward MV of the parity block is obtained.
{
bExistMV = ( C0Avail && getColocatedMVP(pu, REF_PIC_LIST_1, posC0, cColMv, iRefIdx ) )
|| getColocatedMVP( pu, REF_PIC_LIST_1, posC1, cColMv, iRefIdx );
if (bExistMV)
{
dir |= 2;
mrgCtx.mvFieldNeighbours[2 * uiArrayAddr + 1].setMvField(cColMv, iRefIdx);
}
}
if( dir != 0 )
{
bool addTMvp = true;//For true, add TMVP (time domain candidate) to the Merge list
if( addTMvp )
{
mrgCtx.interDirNeighbours[uiArrayAddr] = dir;
#if !JVET_L0090_PAIR_AVG
isCandInter [uiArrayAddr] = true;
#endif
mrgCtx.GBiIdx[uiArrayAddr] = GBI_DEFAULT;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[uiArrayAddr] = false;
#endif
if (mrgCandIdx == cnt && canFastExit)
{
return;
}
cnt++;//Total number of candidates + 1
}
}
}
// early termination
if (cnt == maxNumMergeCand)
{
return;
}
int maxNumMergeCandMin1 = maxNumMergeCand - 1;//Up to five candidates are selected after traversal of candidate blocks in spatial and temporal domains
if (cnt != maxNumMergeCandMin1)//If there are less than five candidates in the current candidate list, continue with the following MVP filling, HMVP and spatial average candidates
{
bool isAvailableSubPu = false;
unsigned subPuMvpPos = 0;
#if JVET_L0090_PAIR_AVG
bool isShared = false;
//The next step is the construction of HMVP candidates, and then the available candidates in HMVP are filled in the Merge list from back to front in turn.
bool bFound = addMergeHMVPCand(cs, mrgCtx, canFastExit
, mrgCandIdx
, maxNumMergeCandMin1, cnt
, spatialCandPos
, isAvailableSubPu, subPuMvpPos
, CU::isIBC(*pu.cu)
, isShared
);
#else
bool bFound = addMergeHMVPCand(slice, mrgCtx, isCandInter, canFastExit
, (mmvdList != 0 && mrgCandIdx != -1) ? (const int)mrgCandIdxIBC : mrgCandIdx
, maxNumMergeCandMin1, cnt, cnt, isAvailableSubPu, subPuMvpPos
, mmvdList
);
#endif
if (bFound)
{
return;
}
}
#if JVET_L0090_PAIR_AVG
// pairwise-average candidates
//HMVP is followed by paired average candidates (i.e. combined average candidates)
{
//If the Merge list has not been filled, then the combination averaging candidate process is performed.
if (cnt > 1 && cnt < maxNumMergeCand)
{
mrgCtx.mvFieldNeighbours[cnt * 2].setMvField( Mv( 0, 0 ), NOT_VALID );//The forward MV of the current candidate block is initialized with zero MV
mrgCtx.mvFieldNeighbours[cnt * 2 + 1].setMvField( Mv( 0, 0 ), NOT_VALID );//Initialize the backward MV of the current candidate block with zero MV
// calculate average MV for L0 and L1 seperately
//Calculating the average MV of L0 and L1
unsigned char interDir = 0;
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[cnt] = (mrgCtx.useAltHpelIf[0] == mrgCtx.useAltHpelIf[1]) ? mrgCtx.useAltHpelIf[0] : false;
#endif
for( int refListId = 0; refListId < (slice.isInterB() ? 2 : 1); refListId++ )
{
const short refIdxI = mrgCtx.mvFieldNeighbours[0 * 2 + refListId].refIdx;//MV Index of the First Candidate Block
const short refIdxJ = mrgCtx.mvFieldNeighbours[1 * 2 + refListId].refIdx;//MV Index of the Second Candidate Block
// both MVs are invalid, skip
if( (refIdxI == NOT_VALID) && (refIdxJ == NOT_VALID) )
{
continue;
}
interDir += 1 << refListId;
// both MVs are valid, average these two MVs
if( (refIdxI != NOT_VALID) && (refIdxJ != NOT_VALID) )
{
const Mv& MvI = mrgCtx.mvFieldNeighbours[0 * 2 + refListId].mv;//MV of the first candidate block
const Mv& MvJ = mrgCtx.mvFieldNeighbours[1 * 2 + refListId].mv;//MV of the second candidate block
// average two MVs
//Average MV of two candidate blocks to get a new MV
Mv avgMv = MvI;
avgMv += MvJ;
roundAffineMv(avgMv.hor, avgMv.ver, 1);
mrgCtx.mvFieldNeighbours[cnt * 2 + refListId].setMvField( avgMv, refIdxI );//The combined new MV is assigned to the current candidate MV.
}
// only one MV is valid, take the only one MV
//If only one candidate block MV is valid, only the MV is used.
else if( refIdxI != NOT_VALID )
{
Mv singleMv = mrgCtx.mvFieldNeighbours[0 * 2 + refListId].mv;
mrgCtx.mvFieldNeighbours[cnt * 2 + refListId].setMvField( singleMv, refIdxI );
}
else if( refIdxJ != NOT_VALID )
{
Mv singleMv = mrgCtx.mvFieldNeighbours[1 * 2 + refListId].mv;
mrgCtx.mvFieldNeighbours[cnt * 2 + refListId].setMvField( singleMv, refIdxJ );
}
}
mrgCtx.interDirNeighbours[cnt] = interDir;
if( interDir > 0 )
{
cnt++;//Total number of MV candidates + 1
}
}
// early termination
if( cnt == maxNumMergeCand )
{
return;
}
}
#endif
uint32_t uiArrayAddr = cnt;
int iNumRefIdx = slice.isInterB() ? std::min(slice.getNumRefIdx(REF_PIC_LIST_0), slice.getNumRefIdx(REF_PIC_LIST_1)) : slice.getNumRefIdx(REF_PIC_LIST_0);
int r = 0;
int refcnt = 0;
//If you haven't filled the Merge list yet, use zero MV to do the final filling.
while (uiArrayAddr < maxNumMergeCand)
{
#if !JVET_L0090_PAIR_AVG
isCandInter [uiArrayAddr ] = true;
#endif
mrgCtx.interDirNeighbours [uiArrayAddr ] = 1;
mrgCtx.GBiIdx [uiArrayAddr ] = GBI_DEFAULT;
mrgCtx.mvFieldNeighbours [uiArrayAddr << 1].setMvField(Mv(0, 0), r);
#if JVET_O0057_ALTHPELIF
mrgCtx.useAltHpelIf[uiArrayAddr] = false;
#endif
if (slice.isInterB())
{
mrgCtx.interDirNeighbours [ uiArrayAddr ] = 3;
mrgCtx.mvFieldNeighbours [(uiArrayAddr << 1) + 1].setMvField(Mv(0, 0), r);
}
if ( mrgCtx.interDirNeighbours[uiArrayAddr] == 1 && pu.cs->slice->getRefPic(REF_PIC_LIST_0, mrgCtx.mvFieldNeighbours[uiArrayAddr << 1].refIdx)->getPOC() == pu.cs->slice->getPOC())
{
mrgCtx.mrgTypeNeighbours[uiArrayAddr] = MRG_TYPE_IBC;
}
uiArrayAddr++;
if (refcnt == iNumRefIdx - 1)
{
r = 0;
}
else
{
++r;
++refcnt;
}
}
mrgCtx.numValidMergeCand = uiArrayAddr;
}