fix: resolve UBSAN and ASAN reported undefined behavior and bugs

source/common/bitstream.cpp

@@ -56,11 +56,7 @@ void Bitstream::write(uint32_t val, uint32_t numBits)
     {
         /* topword aligns m_partialByte with the msb of val */
         uint32_t topword = (numBits - nextPartialBits) & ~7;
-#if USING_FTRAPV
-        uint32_t write_bits = (topword < 32 ? m_partialByte << topword : 0) | (val >> nextPartialBits);
-#else
-        uint32_t write_bits = (m_partialByte << topword) | (val >> nextPartialBits);
-#endif
+        uint32_t write_bits = (topword < 32 ? (uint32_t)m_partialByte << topword : 0) | (val >> nextPartialBits);
 
         switch (writeBytes)
         {

source/common/cudata.cpp

@@ -462,7 +462,8 @@ void CUData::initLosslessCU(const CUData& cu, const CUGeom& cuGeom)
     m_cuAboveRight = cu.m_cuAboveRight;
     m_absIdxInCTU  = cuGeom.absPartIdx;
     m_numPartitions = cuGeom.numPartitions;
-    memcpy(m_qp, cu.m_qp, BytesPerPartition * m_numPartitions);
+    uint32_t bytesPerPartition = (m_chromaFormat == X265_CSP_I400 ? BytesPerPartition - 4 : BytesPerPartition);
+    memcpy(m_qp, cu.m_qp, bytesPerPartition * m_numPartitions);
     memcpy(m_mv[0],  cu.m_mv[0],  m_numPartitions * sizeof(MV));
     memcpy(m_mv[1],  cu.m_mv[1],  m_numPartitions * sizeof(MV));
     memcpy(m_mvd[0], cu.m_mvd[0], m_numPartitions * sizeof(MV));

source/common/cudata.h

@@ -304,8 +304,8 @@ class CUData
     void     getNeighbourMV(uint32_t puIdx, uint32_t absPartIdx, InterNeighbourMV* neighbours) const;
     void     getIntraTUQtDepthRange(uint32_t tuDepthRange[2], uint32_t absPartIdx) const;
     void     getInterTUQtDepthRange(uint32_t tuDepthRange[2], uint32_t absPartIdx) const;
-    uint32_t getBestRefIdx(uint32_t subPartIdx) const { return ((m_interDir[subPartIdx] & 1) << m_refIdx[0][subPartIdx]) | 
-                                                              (((m_interDir[subPartIdx] >> 1) & 1) << (m_refIdx[1][subPartIdx] + 16)); }
+    uint32_t getBestRefIdx(uint32_t subPartIdx) const { return ((uint32_t)(m_interDir[subPartIdx] & 1) << (m_refIdx[0][subPartIdx] & 31)) |
+                                                              ((uint32_t)((m_interDir[subPartIdx] >> 1) & 1) << ((m_refIdx[1][subPartIdx] + 16) & 31)); }
     uint32_t getPUOffset(uint32_t puIdx, uint32_t absPartIdx) const { return (partAddrTable[(int)m_partSize[absPartIdx]][puIdx] << (m_slice->m_param->unitSizeDepth - m_cuDepth[absPartIdx]) * 2) >> 4; }
 
     uint32_t getNumPartInter(uint32_t absPartIdx) const              { return nbPartsTable[(int)m_partSize[absPartIdx]]; }

source/common/dct.cpp

@@ -658,7 +658,7 @@ static void dequant_scaling_c(const int16_t* quantCoef, const int32_t* deQuantCo
         for (int n = 0; n < num; n++)
         {
             coeffQ   = x265_clip3(-32768, 32767, quantCoef[n] * deQuantCoef[n]);
-            coef[n] = (int16_t)x265_clip3(-32768, 32767, coeffQ << (per - shift));
+            coef[n] = (int16_t)x265_clip3(-32768, 32767, coeffQ * (1 << (per - shift)));
         }
     }
 }

source/common/deblock.cpp

@@ -325,8 +325,8 @@ void Deblock::edgeFilterLuma(const CUData* cuQ, uint32_t absPartIdx, uint32_t de
 
     int32_t maskP = -1;
     int32_t maskQ = -1;
-    int32_t betaOffset = pps->deblockingFilterBetaOffsetDiv2 << 1;
-    int32_t tcOffset = pps->deblockingFilterTcOffsetDiv2 << 1;
+    int32_t betaOffset = pps->deblockingFilterBetaOffsetDiv2 * 2;
+    int32_t tcOffset = pps->deblockingFilterTcOffsetDiv2 * 2;
     bool bCheckNoFilter = pps->bTransquantBypassEnabled;
 
     if (dir == EDGE_VER)
@@ -422,7 +422,7 @@ void Deblock::edgeFilterChroma(const CUData* cuQ, uint32_t absPartIdx, uint32_t
 
     int32_t maskP = -1;
     int32_t maskQ = -1;
-    int32_t tcOffset = pps->deblockingFilterTcOffsetDiv2 << 1;
+    int32_t tcOffset = pps->deblockingFilterTcOffsetDiv2 * 2;
 
     X265_CHECK(((dir == EDGE_VER)
                 ? ((g_zscanToPelX[absPartIdx] + edge * UNIT_SIZE) >> cuQ->m_hChromaShift)

source/common/mv.h

@@ -56,14 +56,8 @@ struct MV
 
     MV& operator >>=(int i)                    { x >>= i; y >>= i; return *this; }
 
-#if USING_FTRAPV
-    /* avoid signed left-shifts when -ftrapv is enabled */
-    MV& operator <<=(int i)                    { x *= (1 << i); y *= (1 << i); return *this; }
-    MV operator <<(int i) const                { return MV(x * (1 << i), y * (1 << i)); }
-#else
-    MV& operator <<=(int i)                    { x <<= i; y <<= i; return *this; }
-    MV operator <<(int i) const                { return MV(x << i, y << i); }
-#endif
+    MV& operator <<=(int i)                    { x = (int32_t)((uint32_t)x << i); y = (int32_t)((uint32_t)y << i); return *this; }
+    MV operator <<(int i) const                { return MV((int32_t)((uint32_t)x << i), (int32_t)((uint32_t)y << i)); }
 
     MV operator >>(int i) const                { return MV(x >> i, y >> i); }
 

source/common/picyuv.cpp

@@ -67,6 +67,10 @@ PicYuv::PicYuv()
     m_strideC = 0;
     m_hChromaShift = 0;
     m_vChromaShift = 0;
+    m_lumaMarginX = 0;
+    m_lumaMarginY = 0;
+    m_chromaMarginX = 0;
+    m_chromaMarginY = 0;
 }
 
 bool PicYuv::create(x265_param* param, bool picAlloc, pixel *pixelbuf)

source/common/pixel.cpp

@@ -400,7 +400,7 @@ void cpy2Dto1D_shl(int16_t* dst, const int16_t* src, intptr_t srcStride, int shi
     for (int i = 0; i < size; i++)
     {
         for (int j = 0; j < size; j++)
-            dst[j] = src[j] << shift;
+            dst[j] = (int16_t)((uint32_t)src[j] << shift);
 
         src += srcStride;
         dst += size;
@@ -435,7 +435,7 @@ void cpy1Dto2D_shl(int16_t* dst, const int16_t* src, intptr_t dstStride, int shi
     for (int i = 0; i < size; i++)
     {
         for (int j = 0; j < size; j++)
-            dst[j] = src[j] << shift;
+            dst[j] = (int16_t)((uint32_t)src[j] << shift);
 
         src += size;
         dst += dstStride;
@@ -629,8 +629,8 @@ static void ssim_4x4x2_core(const pixel* pix1, intptr_t stride1, const pixel* pi
         {
             for (int x = 0; x < 4; x++)
             {
-                int a = pix1[x + y * stride1];
-                int b = pix2[x + y * stride2];
+                uint32_t a = pix1[x + y * stride1];
+                uint32_t b = pix2[x + y * stride2];
                 s1 += a;
                 s2 += b;
                 ss += a * a;

source/common/predict.cpp

@@ -342,8 +342,8 @@ void Predict::predInterChromaPixel(const PredictionUnit& pu, Yuv& dstYuv, const
     intptr_t dstStride = dstYuv.m_csize;
     intptr_t refStride = refPic.m_strideC;
 
-    int mvx = mv.x << (1 - m_hChromaShift);
-    int mvy = mv.y << (1 - m_vChromaShift);
+    int mvx = (int32_t)((uint32_t)mv.x << (1 - m_hChromaShift));
+    int mvy = (int32_t)((uint32_t)mv.y << (1 - m_vChromaShift));
 
     intptr_t refOffset = (mvx >> 3) + (mvy >> 3) * refStride;
 
@@ -391,8 +391,8 @@ void Predict::predInterChromaShort(const PredictionUnit& pu, ShortYuv& dstSYuv,
     intptr_t dstStride = dstSYuv.m_csize;
     intptr_t refStride = refPic.m_strideC;
 
-    int mvx = mv.x << (1 - m_hChromaShift);
-    int mvy = mv.y << (1 - m_vChromaShift);
+    int mvx = (int32_t)((uint32_t)mv.x << (1 - m_hChromaShift));
+    int mvy = (int32_t)((uint32_t)mv.y << (1 - m_vChromaShift));
 
     intptr_t refOffset = (mvx >> 3) + (mvy >> 3) * refStride;
 

source/common/primitives.cpp

@@ -86,28 +86,99 @@ void enableLowpassDCTPrimitives(EncoderPrimitives &p)
     p.cu[BLOCK_32x32].dct = p.cu[BLOCK_32x32].lowpass_dct;
 }
 
-void setupAliasPrimitives(EncoderPrimitives &p)
-{
 #if HIGH_BIT_DEPTH
-    /* at HIGH_BIT_DEPTH, pixel == short so we can alias many primitives */
-    for (int i = 0; i < NUM_CU_SIZES; i++)
+namespace {
+/* At HIGH_BIT_DEPTH pixel and int16_t are the same-size 16-bit type, so the
+ * short primitives can serve the pixel variants. These trampolines reinterpret
+ * only the data pointers, avoiding the UB of casting the function pointers
+ * themselves (flagged by UBSan's -fsanitize=function). */
+
+#if !defined(X265_ARCH_ARM64) && !defined(X265_ARCH_RISCV64)
+template<int i>
+sse_t aliasSsePP(const pixel* a, intptr_t strideA, const pixel* b, intptr_t strideB)
+{
+    return primitives.cu[i].sse_ss(reinterpret_cast<const int16_t*>(a), strideA,
+                                   reinterpret_cast<const int16_t*>(b), strideB);
+}
+#endif
+
+template<int i>
+void aliasLumaCopyPS(int16_t* dst, intptr_t dstStride, const pixel* src, intptr_t srcStride)
+{   primitives.pu[i].copy_pp(reinterpret_cast<pixel*>(dst), dstStride, src, srcStride); }
+
+template<int i>
+void aliasLumaCopySP(pixel* dst, intptr_t dstStride, const int16_t* src, intptr_t srcStride)
+{   primitives.pu[i].copy_pp(dst, dstStride, reinterpret_cast<const pixel*>(src), srcStride); }
+
+template<int i>
+void aliasLumaCopySS(int16_t* dst, intptr_t dstStride, const int16_t* src, intptr_t srcStride)
+{   primitives.pu[i].copy_pp(reinterpret_cast<pixel*>(dst), dstStride, reinterpret_cast<const pixel*>(src), srcStride); }
+
+template<int csp, int i>
+void aliasChromaCopyPS(int16_t* dst, intptr_t dstStride, const pixel* src, intptr_t srcStride)
+{   primitives.chroma[csp].pu[i].copy_pp(reinterpret_cast<pixel*>(dst), dstStride, src, srcStride); }
+
+template<int csp, int i>
+void aliasChromaCopySP(pixel* dst, intptr_t dstStride, const int16_t* src, intptr_t srcStride)
+{   primitives.chroma[csp].pu[i].copy_pp(dst, dstStride, reinterpret_cast<const pixel*>(src), srcStride); }
+
+template<int csp, int i>
+void aliasChromaCopySS(int16_t* dst, intptr_t dstStride, const int16_t* src, intptr_t srcStride)
+{   primitives.chroma[csp].pu[i].copy_pp(reinterpret_cast<pixel*>(dst), dstStride, reinterpret_cast<const pixel*>(src), srcStride); }
+
+template<int i>
+struct AliasFiller
+{
+    static void fill(EncoderPrimitives& p)
     {
+        /* Only install an alias when the underlying short primitive it forwards
+         * to actually exists. The trampolines dispatch through the base entry,
+         * so aliasing a missing base would produce a non-NULL pointer that jumps
+         * through a NULL slot when invoked. Mirroring the base's presence keeps
+         * the original "NULL base => NULL alias" behaviour that callers (and the
+         * testbench's "if (opt.foo)" guards) rely on. */
 #if !defined(X265_ARCH_ARM64) && !defined(X265_ARCH_RISCV64)
-        p.cu[i].sse_pp = (pixel_sse_t)p.cu[i].sse_ss;
+        if (p.cu[i].sse_ss)
+            p.cu[i].sse_pp = aliasSsePP<i>;
 #endif
+        if (p.pu[i].copy_pp)
+        {
+            p.cu[i].copy_ps = aliasLumaCopyPS<i>;
+            p.cu[i].copy_sp = aliasLumaCopySP<i>;
+            p.cu[i].copy_ss = aliasLumaCopySS<i>;
+        }
 
-        p.cu[i].copy_ps = (copy_ps_t)p.pu[i].copy_pp;
-        p.cu[i].copy_sp = (copy_sp_t)p.pu[i].copy_pp;
-        p.cu[i].copy_ss = (copy_ss_t)p.pu[i].copy_pp;
+        if (p.chroma[X265_CSP_I420].pu[i].copy_pp)
+        {
+            p.chroma[X265_CSP_I420].cu[i].copy_ps = aliasChromaCopyPS<X265_CSP_I420, i>;
+            p.chroma[X265_CSP_I420].cu[i].copy_sp = aliasChromaCopySP<X265_CSP_I420, i>;
+            p.chroma[X265_CSP_I420].cu[i].copy_ss = aliasChromaCopySS<X265_CSP_I420, i>;
+        }
 
-        p.chroma[X265_CSP_I420].cu[i].copy_ps = (copy_ps_t)p.chroma[X265_CSP_I420].pu[i].copy_pp;
-        p.chroma[X265_CSP_I420].cu[i].copy_sp = (copy_sp_t)p.chroma[X265_CSP_I420].pu[i].copy_pp;
-        p.chroma[X265_CSP_I420].cu[i].copy_ss = (copy_ss_t)p.chroma[X265_CSP_I420].pu[i].copy_pp;
+        if (p.chroma[X265_CSP_I422].pu[i].copy_pp)
+        {
+            p.chroma[X265_CSP_I422].cu[i].copy_ps = aliasChromaCopyPS<X265_CSP_I422, i>;
+            p.chroma[X265_CSP_I422].cu[i].copy_sp = aliasChromaCopySP<X265_CSP_I422, i>;
+            p.chroma[X265_CSP_I422].cu[i].copy_ss = aliasChromaCopySS<X265_CSP_I422, i>;
+        }
 
-        p.chroma[X265_CSP_I422].cu[i].copy_ps = (copy_ps_t)p.chroma[X265_CSP_I422].pu[i].copy_pp;
-        p.chroma[X265_CSP_I422].cu[i].copy_sp = (copy_sp_t)p.chroma[X265_CSP_I422].pu[i].copy_pp;
-        p.chroma[X265_CSP_I422].cu[i].copy_ss = (copy_ss_t)p.chroma[X265_CSP_I422].pu[i].copy_pp;
+        AliasFiller<i + 1>::fill(p);
     }
+};
+
+template<>
+struct AliasFiller<NUM_CU_SIZES>
+{
+    static void fill(EncoderPrimitives&) {}
+};
+} // namespace
+#endif
+
+void setupAliasPrimitives(EncoderPrimitives &p)
+{
+#if HIGH_BIT_DEPTH
+    /* at HIGH_BIT_DEPTH, pixel == short so we can alias many primitives */
+    AliasFiller<0>::fill(p);
 #endif
 
     /* alias chroma 4:4:4 from luma primitives (all but chroma filters) */

source/common/quant.cpp

@@ -1341,7 +1341,7 @@ uint32_t Quant::rdoQuant(const CUData& cu, int16_t* dstCoeff, TextType ttype, ui
 
                         const int64_t origDist = (((int64_t)d * d));
 
-#define DELTARDCOST(d0, d, deltabits) ((((int64_t)d * d - d0) << scaleBits) + ((lambda2 * (int64_t)(deltabits)) >> 8))
+#define DELTARDCOST(d0, d, deltabits) ((int64_t)((uint64_t)((int64_t)d * d - d0) << scaleBits) + ((lambda2 * (int64_t)(deltabits)) >> 8))
 
                         const uint32_t isOne = (absLevel == 1);
                         if (dstCoeff[blkPos])

source/common/quant.h

@@ -53,9 +53,10 @@ struct QpParam
             rem = qpScaled % 6;
             per = qpScaled / 6;
             qp  = qpScaled;
-            lambda2 = (int64_t)(x265_lambda2_tab[qp - QP_BD_OFFSET] * 256. + 0.5);
-            lambda  = (int32_t)(x265_lambda_tab[qp - QP_BD_OFFSET] * 256. + 0.5);
-            X265_CHECK((x265_lambda_tab[qp - QP_BD_OFFSET] * 256. + 0.5) < (double)MAX_INT, "x265_lambda_tab[] value too large\n");
+            int lambdaIdx = x265_clip3(QP_MIN, QP_MAX_MAX, qp - QP_BD_OFFSET);
+            lambda2 = (int64_t)(x265_lambda2_tab[lambdaIdx] * 256. + 0.5);
+            lambda  = (int32_t)(x265_lambda_tab[lambdaIdx] * 256. + 0.5);
+            X265_CHECK((x265_lambda_tab[lambdaIdx] * 256. + 0.5) < (double)MAX_INT, "x265_lambda_tab[] value too large\n");
         }
     }
 };
@@ -125,7 +126,7 @@ class Quant
         // NOTE: cgBlkPos+1 may more than 63, it is invalid for shift,
         //       but in this case, both cgPosX and cgPosY equal to (trSizeCG - 1),
         //       the sigRight and sigLower will clear value to zero, the final result will be correct
-        const uint32_t sigPos = (uint32_t)(sigCoeffGroupFlag64 >> (cgBlkPos + 1)); // just need lowest 7-bits valid
+        const uint32_t sigPos = (uint32_t)((sigCoeffGroupFlag64 >> cgBlkPos) >> 1); // just need lowest 7-bits valid
 
         // TODO: instruction BT is faster, but _bittest64 still generate instruction 'BT m, r' in VS2012
         const uint32_t sigRight = (cgPosX != (trSizeCG - 1)) & sigPos;
@@ -138,7 +139,7 @@ class Quant
     {
         X265_CHECK(cgBlkPos < 64, "cgBlkPos is too large\n");
         // NOTE: unsafe shift operator, see NOTE in calcPatternSigCtx
-        const uint32_t sigPos = (uint32_t)(cgGroupMask >> (cgBlkPos + 1)); // just need lowest 8-bits valid
+        const uint32_t sigPos = (uint32_t)((cgGroupMask >> cgBlkPos) >> 1); // just need lowest 8-bits valid
         const uint32_t sigRight = (cgPosX != (trSizeCG - 1)) & sigPos;
         const uint32_t sigLower = (cgPosY != (trSizeCG - 1)) & (sigPos >> (trSizeCG - 1));
 

source/encoder/analysis.cpp

@@ -4381,7 +4381,10 @@ int Analysis::calculateQpforCuSize(const CUData& ctu, const CUGeom& cuGeom, int3
                     cnt++;
                 }
             }
-            dQpOffset /= cnt;
+            if (cnt)
+            {
+                dQpOffset /= cnt;
+            }
             qp += dQpOffset;
             if (complexCheck)
             {

source/encoder/encoder.cpp

@@ -2249,9 +2249,9 @@ int Encoder::encode(const x265_picture* pic_in, x265_picture* pic_out)
 
         /* pop a single frame from decided list, then provide to frame encoder
          * curEncoder is guaranteed to be idle at this point */
-        if (!pass)
+        if (!pass && (!m_param->chunkEnd || (m_encodedFrameNum < m_param->chunkEnd)))
             frameEnc[0] = m_lookahead->getDecidedPicture();
-        if (frameEnc[0] && !pass && (!m_param->chunkEnd || (m_encodedFrameNum < m_param->chunkEnd)))
+        if (frameEnc[0] && !pass)
         {
 
 #if ENABLE_ALPHA || ENABLE_MULTIVIEW

source/encoder/framefilter.cpp

@@ -385,15 +385,21 @@ void FrameFilter::ParallelFilter::processPostCu(int col) const
     if ((col == 0) | (col == m_frameFilter->m_numCols - 1))
     {
         copySizeY += lumaMarginX;
-        copySizeC += chromaMarginX;
+        if (m_frameFilter->m_param->internalCsp != X265_CSP_I400)
+        {
+            copySizeC += chromaMarginX;
+        }
     }
 
     // First column need extension left padding area and first CU
     if (col == 0)
     {
         pixY -= lumaMarginX;
-        pixU -= chromaMarginX;
-        pixV -= chromaMarginX;
+        if (m_frameFilter->m_param->internalCsp != X265_CSP_I400)
+        {
+            pixU -= chromaMarginX;
+            pixV -= chromaMarginX;
+        }
     }
 
     // Border extend Top
@@ -416,8 +422,11 @@ void FrameFilter::ParallelFilter::processPostCu(int col) const
     if (m_row == m_frameFilter->m_numRows - 1)
     {
         pixY += (realH - 1) * stride;
-        pixU += ((realH >> vChromaShift) - 1) * strideC;
-        pixV += ((realH >> vChromaShift) - 1) * strideC;
+        if (m_frameFilter->m_param->internalCsp != X265_CSP_I400)
+        {
+            pixU += ((realH >> vChromaShift) - 1) * strideC;
+            pixV += ((realH >> vChromaShift) - 1) * strideC;
+        }
         for (uint32_t y = 0; y < lumaMarginY; y++)
             memcpy(pixY + (y + 1) * stride, pixY, copySizeY * sizeof(pixel));