Oodle Texture BC7RD "preserve extremes" constraint validation

gistfile1.txt

//===================================================================
// constraint validation for preserve extremes mode

// Cold part of are_endpoitns_permitted: the actual constraint validation
static RADNOINLINE bool are_endpoints_permitted_cold(const BC7BlockState & st, const bc7rd_blockinfo& info)
{
	const BC7Flags flags = info.flags;

	if (st.mode <= 3)
	{
		// Modes 0-3 (alpha always =255) only legal if we only have A=255 constrained pixels
		// since A=0 constraints can never be satisfied
		return (flags & BC7ENC_PRESERVE_A0) == 0;
	}
	else if (st.mode <= 5)
	{
		// Modes 4 and 5. Check whether min/max alpha obey our constraints.
		const BC7SubsetState& sst0 = st.subsets[0];

		// Constrained blocks must have alpha as the scalar channel
		if (sst0.rbit != 0)
			return false;

		const U8 a0 = sst0.endpoints[0].a;
		const U8 a1 = sst0.endpoints[1].a;

		if ((flags & BC7ENC_PRESERVE_A0) && RR_MIN(a0, a1) != 0)
			return false;
		if ((flags & BC7ENC_PRESERVE_A255) && RR_MAX(a0, a1) != 255)
			return false;

		// If the first pixel in block (=anchor pixel) is constrained, need to make sure
		// the endpoint ordering is such that the constraint is satisfied, since we can't
		// swap endpoints without losing the endpoint reuse that's the whole point
		if ( info.constraints.mask & 0x00010001u )
		{
			// implied index MSB at anchor pixel is always 0
			// -> if constrained lo, a0 needs to be 0;
			// if constrained hi, a0 needs to be 255
			return a0 == ((info.constraints.mask & 1) ? 0 : 255);
		}

		return true;
	}
	else
	{
		RR_ASSERT(st.mode <= 7);

		// Modes 6 and 7. We only allow this if both endpoint A values
		// are equal and match the constraint value.
		//
		// If both A=0 and A=255 are present, this is never legal
		if ((flags & BC7ENC_PRESERVE_EXTREMES) == BC7ENC_PRESERVE_EXTREMES)
			return false;

		const U8 need_a = (flags & BC7ENC_PRESERVE_A0) ? 0 : 255;

		// Check first subset always
		const BC7SubsetState& sst0 = st.subsets[0];
		if (sst0.endpoints[0].a != need_a || sst0.endpoints[1].a != need_a)
			return false;

		// Mode 7 also has a second subset that needs to be checked
		if (st.mode == 7)
		{
			const BC7SubsetState& sst1 = st.subsets[1];
			if (sst1.endpoints[0].a != need_a || sst1.endpoints[1].a != need_a)
				return false;
		}

		return true;
	}
}

// Hot path is just the "preserve extremes" check, which is usually off. This part should get inlined.
static RADFORCEINLINE bool are_endpoints_permitted(const BC7BlockState & st, const bc7rd_blockinfo& info)
{
	// If no preserve extremes, we don't have particular endpoint constraints
	if ((info.flags & BC7ENC_PRESERVE_EXTREMES) == 0)
		return true;

	return are_endpoints_permitted_cold(st, info);
}

// Can we do an index substitution? Only one caller, so might as well forceinline.
static RADFORCEINLINE bool are_indices_permitted(const BC7BlockState & st, const bc7rd_blockinfo& info, BC7Flags *pFlags)
{
	*pFlags = info.flags;
	BC7Flags flags = info.flags;

	// If no preserves extremes, we don't have particular index contraints
	if ((flags & BC7ENC_PRESERVE_EXTREMES) == 0)
		return true;

	if (st.mode <= 3)
	{
		// Modes 0-3 (alpha always =255) only legal if we only have A=255 constrained pixels
		// since A=0 constraints can never be satisfied, but no actual index constraints
		return (flags & BC7ENC_PRESERVE_A0) == 0;
	}
	else if (st.mode <= 5)
	{
		// Modes 4 and 5. In this case, we have index constraints.
		const BC7SubsetState& sst = st.subsets[0];

		// Constrained blocks must have alpha as the scalar channel
		if (sst.rbit != 0)
			return false;

		// Grab the alpha indices
		const BC7Inds& a_inds = sst.idxs[sst.isbit ^ 1];

		// Figure out extremal index values; we don't care which decode to A=0 or A=255,
		// since we determine new endpoints after, the two can switch places, so either
		// works.
		const int ib = (st.mode == 5 || sst.isbit == 1) ? 2 : 3; // Mode 4 ib2 is 3 bits, else 2 bits index.
		BC7Constraints satisfiable = BC7Constraints::from_indices(a_inds, ib);

		if (satisfiable.superset_of(info.constraints))
		{
			// Satisfiable with min_ind->0 and max_ind->255 ordering
			*pFlags = flags | BC7ENC_AEP0_LO;
			return true;
		}
		else if (satisfiable.flip().superset_of(info.constraints))
		{
			// Satisfiable with min_ind->255 and max_ind->0 ordering
			*pFlags = flags | BC7ENC_AEP0_HI;
			return true;
		}

		return false;
	}
	else
	{
		RR_ASSERT(st.mode <= 7);

		// Modes 6 and 7. We only use this for constrained blocks when A is constant.
		// We only use them for A=constant blocks and pick feasible endpoints, so
		// there are no actual index constraints here. If mode 6/7 were allowed, any
		// indices work. In mode 7, the pbits need to be compatible as well.
		if (flags & BC7ENC_DISABLE_MODE67)
			return false;

		if (st.mode != 7)
			return true;

		// In mode 7, the pbits for both subsets need to be all 0 when we're trying
		// to hit A=0 and all 1 when we're trying to hit A=255.
		U8 need_pbits = (flags & BC7ENC_PRESERVE_A255) ? 3 : 0;
		return st.subsets[0].pbits == need_pbits && st.subsets[1].pbits == need_pbits;
	}
}