Hardcoded qp clipping

[gakssultang]

Summary

MAX_QUANT(BD) is bit-depth-aware (src/oapv_def.h:55) so the QP ceiling tracks bit depth:

#define MAX_QUANT(BD)  (63 + ((BD-10)*6))   // 10-bit -> 63, 12-bit -> 75

But the encoder validates/clips QP against a hardcoded MAX_QUANT(10) (= 63) everywhere, so the top of the 12-bit quantization range (QP 64–75) is unreachable.

Version: v0.3.0.0

Fix

The correct ceiling is always MAX_QUANT(<internal bit depth>) = MAX_QUANT(ctx->bit_depth).

1. src/oapv.c — enc_frm_prepare (needs reordering)

-    // line 837 — runs before bit depth is known
-    oapv_assert_rv((param->qp >= MIN_QUANT && param->qp <= MAX_QUANT(10)) || param->qp == OAPVE_PARAM_QP_AUTO, OAPV_ERR_INVALID_QP);
-    ...
-    // line 868
-    for(i = 0; i < N_C; i++) {
-        ctx->qp[i] = oapv_clip3(MIN_QUANT, MAX_QUANT(10), param->qp + ctx->qp_offset[i]);
-    }

     // ... ctx->bit_depth_inp = ...; ret = enc_check_profile(...); ... ctx->bit_depth = ...;
+
+    // perform both AFTER ctx->bit_depth is assigned (after line 885)
+    oapv_assert_rv((param->qp >= MIN_QUANT && param->qp <= MAX_QUANT(ctx->bit_depth)) || param->qp == OAPVE_PARAM_QP_AUTO, OAPV_ERR_INVALID_QP);
+
+    for(i = 0; i < N_C; i++) {
+        ctx->qp[i] = oapv_clip3(MIN_QUANT, MAX_QUANT(ctx->bit_depth), param->qp + ctx->qp_offset[i]);
+    }

2. src/oapv.c:1005 — oapve_encode

-    ctx->qp[c] = oapv_clip3(MIN_QUANT, MAX_QUANT(10), ctx->rc_param.qp + ctx->qp_offset[c]);
+    ctx->qp[c] = oapv_clip3(MIN_QUANT, MAX_QUANT(ctx->bit_depth), ctx->rc_param.qp + ctx->qp_offset[c]);

3. src/oapv.c:1258 — oapve_config OAPV_CFG_SET_QP

-    // if ctx->bit_depth is set at config time
-    oapv_assert_rv(t0 >= MIN_QUANT && t0 <= MAX_QUANT(10), OAPV_ERR_INVALID_ARGUMENT);
+    oapv_assert_rv(t0 >= MIN_QUANT && t0 <= MAX_QUANT(ctx->bit_depth), OAPV_ERR_INVALID_ARGUMENT);

4. src/oapv_rc.c:169 — oapve_rc_estimate_pic_qp (needs bit_depth parameter)

This function has no ctx, so thread the bit depth in.

 // src/oapv_rc.h:44  — declaration
-int oapve_rc_estimate_pic_qp(double lambda);
+int oapve_rc_estimate_pic_qp(double lambda, int bit_depth);

 // src/oapv_rc.c:166
-int oapve_rc_estimate_pic_qp(double lambda)
+int oapve_rc_estimate_pic_qp(double lambda, int bit_depth)
 {
     int qp = (int)(4.2005 * log(lambda) + 13.7122 + 0.5) + OAPV_RC_QP_OFFSET;
-    qp = oapv_clip3(MIN_QUANT, MAX_QUANT(10), qp);
+    qp = oapv_clip3(MIN_QUANT, MAX_QUANT(bit_depth), qp);
     return qp;
 }

 // src/oapv.c:998  — caller
-    ctx->rc_param.qp = oapve_rc_estimate_pic_qp(ctx->rc_param.lambda);
+    ctx->rc_param.qp = oapve_rc_estimate_pic_qp(ctx->rc_param.lambda, ctx->bit_depth);

5. src/oapv_rc.c:196 — oapve_rc_get_qp

-    *qp = oapv_clip3(MIN_QUANT, MAX_QUANT(10), *qp);
+    *qp = oapv_clip3(MIN_QUANT, MAX_QUANT(ctx->bit_depth), *qp);

6. src/oapv_vlc.c:323 — oapve_set_tile_header

-    th->tile_qp[c] = oapv_clip3(MIN_QUANT, MAX_QUANT(10), qp + ctx->qp_offset[c]);
+    th->tile_qp[c] = oapv_clip3(MIN_QUANT, MAX_QUANT(ctx->bit_depth), qp + ctx->qp_offset[c]);