#######################################################################
# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
# All rights reserved.
#
# This source code is licensed under a BSD-style license (found in the
# LICENSE file in the root directory of this source tree)
#######################################################################

"""Indexing support mixed into :class:`blosc2.CTable`."""

from __future__ import annotations

import ast
import contextlib
import os
import pathlib
from typing import TYPE_CHECKING, Any

import numpy as np

import blosc2
from blosc2 import compute_chunks_blocks
from blosc2.ctable_nulls import NULL_MASK, NULL_SENTINEL, is_nan_sentinel, kind_of_spec, sentinel_mask
from blosc2.schema import (
    DictionarySpec,
    ListSpec,
    NDArraySpec,
    ObjectSpec,
    StructSpec,
    UTF8Spec,
    VLBytesSpec,
    VLStringSpec,
)

if TYPE_CHECKING:
    from blosc2.ctable import CTable


class _FakeVlMeta:
    """Minimal vlmeta stand-in that accepts writes without touching a real SChunk."""

    def __init__(self):
        self._data: dict = {}

    def __getitem__(self, key):
        return self._data[key]

    def __setitem__(self, key, value):
        self._data[key] = value

    def get(self, key, default=None):
        return self._data.get(key, default)


class _FakeSchunk:
    """Minimal SChunk stand-in whose vlmeta stores in memory."""

    def __init__(self):
        self.vlmeta = _FakeVlMeta()


def _dict_code_to_rank(dictionary) -> np.ndarray:
    """``code -> alphabetical rank`` lookup for a dictionary column.

    Sorting by rank is sorting by decoded value, which is what lets an int32
    array stand in for the strings — in the FULL index (:class:`_DictRankWrapper`)
    and in ``CTable._build_lex_keys``.  The reserved null code is *not* a
    dictionary entry, so callers assign nulls a rank of their own (``len``,
    the largest, so nulls sort last).
    """
    n_entries = len(dictionary)
    order = np.argsort(dictionary, kind="stable")
    code_to_rank = np.empty(n_entries, dtype=np.int32)
    code_to_rank[order] = np.arange(n_entries, dtype=np.int32)
    return code_to_rank


def _dict_rank_hash(dictionary) -> str:
    """Stable hash of a dictionary's entries (code position + value).

    Used to detect when a rank-based FULL index has gone stale (the alphabetical
    ranks it encodes no longer match the live dictionary).  Must be stable across
    processes — the hash is persisted in the index descriptor and recomputed on a
    fresh open; ``hash()`` is PYTHONHASHSEED-salted and would spuriously mismatch,
    making the persistent index always fall back.
    """
    import hashlib

    h = hashlib.sha1(usedforsecurity=False)
    for i, value in enumerate(dictionary):
        h.update(repr((i, value)).encode("utf-8"))
    return h.hexdigest()


#: Rows factorized per pass when building a utf8 rank index.  Bounds the
#: transient code buffer without changing the result.
_UTF8_RANK_SPAN = 1 << 20


def _utf8_rank_arrays(col, n_phys: int, null_value: str | None, *, valid=None):
    """Alphabetical rank per row for a utf8 column, plus its staleness metadata.

    Sorting by rank is sorting by decoded string, so an ``int32`` rank column
    can drive the whole numeric index machinery unchanged — the same trick
    :class:`_DictRankWrapper` plays for dictionary columns.  Unlike a dictionary
    there is no stored code array, so the column is factorized here; the
    factorizer hashes raw bytes and only ever decodes the distinct values.

    Under sentinel storage null rows carry a sentinel *string*, so the sentinel
    is just another vocabulary entry; it is given the largest rank so nulls sort
    last, matching both the dictionary index and ``_build_lex_keys``.

    Under mask storage there is **no sentinel in the vocabulary**: a null row
    holds the empty-string fill, which factorizes as an ordinary entry with rank
    0 — so nulls would sort *first* and be indistinguishable from genuine ``""``
    rows in every rank comparison the index answers.  *valid* (physical, one
    flag per row) is what closes that hole: null rows are stamped with
    ``null_rank`` after the rank gather, exactly where the sentinel path would
    have put them.
    """
    fact = col.factorizer()
    codes = np.empty(n_phys, dtype=np.int64)
    for start in range(0, n_phys, _UTF8_RANK_SPAN):
        stop = min(start + _UTF8_RANK_SPAN, n_phys)
        codes[start:stop] = fact.codes_for_span(start, stop)
    uniques = fact.uniques()
    n_entries = len(uniques)

    is_null = uniques == null_value if null_value is not None else np.zeros(n_entries, dtype=bool)
    non_null = np.flatnonzero(~is_null)
    order = non_null[np.argsort(uniques[non_null], kind="stable")]
    code_to_rank = np.empty(max(n_entries, 1), dtype=np.int32)
    code_to_rank[order] = np.arange(len(order), dtype=np.int32)
    null_rank = np.int32(len(order))
    if is_null.any():
        code_to_rank[is_null] = null_rank

    # Rank order == alphabetical order, so this doubles as the lookup table that
    # turns a query literal into a rank (np.searchsorted) without touching data.
    sorted_vocab = uniques[order]
    ranks = code_to_rank[codes] if n_entries else np.zeros(n_phys, dtype=np.int32)
    if valid is not None:
        # Mask storage: after the gather, not before.  The fill in a null row is
        # a legitimate vocabulary entry that other rows may share, so the
        # rewrite has to be per row rather than per vocabulary entry.
        ranks = np.where(np.asarray(valid[:n_phys], dtype=bool), ranks, null_rank)
    # Staleness signals must be O(1) to check: re-deriving the vocabulary would
    # mean factorizing the column again on every query.  Any write already marks
    # every index stale, so these only have to catch a rebuilt-but-changed
    # column, for which row count plus blob size is enough.
    meta = {
        "null_rank": int(null_rank),
        "null_aware": null_value is not None or valid is not None,
        "vocab_len": int(n_entries),
        "n_rows": int(n_phys),
        "nbytes": int(col._bytes_used),
    }
    return ranks.astype(np.int32, copy=False), meta, sorted_vocab


def _persist_utf8_vocab(full: dict, meta: dict, sorted_vocab: np.ndarray) -> None:
    """Store the rank-ordered vocabulary so a query literal can be turned into a rank.

    Written beside the index's own sidecars when the table is persistent, and
    inlined into the descriptor otherwise — the in-memory index path is for
    small tables by construction.  Without it a literal→rank lookup would mean
    factorizing the column again on every query.
    """
    if len(sorted_vocab) == 0:
        meta["vocab"] = []
        return
    values_path = full.get("values_path")
    if values_path is None:  # in-memory index
        meta["vocab"] = sorted_vocab.tolist()
        return
    # max(1, ...): a mask-storage column's ``""`` fill is an ordinary vocabulary
    # entry, so an all-null column factorizes to exactly ``[""]`` -- and ``<U0``
    # is not a dtype NumPy will cast a StringDType array into.
    width = max(1, max(len(v) for v in sorted_vocab))
    vocab_path = str(pathlib.Path(values_path).with_suffix("")) + ".utf8_vocab.b2nd"
    blosc2.asarray(sorted_vocab.astype(f"<U{width}"), urlpath=vocab_path, mode="w")
    meta["vocab_path"] = vocab_path


def _expression_defeats_global_null_filter(expression: str) -> bool:
    """Whether *expression* really contains a boolean OR, or a negation.

    Both make a *global* null post-filter wrong, for the same underlying
    reason: the predicate's answer for a null row is not simply "no match".
    Under OR a row that is null in one column can still match the other
    branch; under Kleene negation ``~(unknown & false)`` is *true*, so a row
    the filter would drop qualifies (see
    ``blosc2.ctable_nulls._NullPredicateRewriter._channels``).  Where the
    predicate is actually evaluated -- the segment path -- both are already
    exact per leaf and the filter is not merely wrong but unnecessary.

    A substring test for ``"|"`` also fires on a string literal that happens to
    contain one (``name == 'a|b'``), and the answer decides whether a nullable
    indexed column may keep its index -- so it is worth parsing for.
    Unparseable input answers True, which is the conservative side.

    **Not every negation is a user's.** ``Column._null_aware_compare`` builds a
    sentinel column's guard as ``result & ~<null predicate>``, so *every*
    operator-form predicate over one carries a ``~`` that the user never wrote:
    ``(t.a > 90)`` is ``((o0 > 90) & ~((o0 == -1)))``.  Answering True for it
    took the index away from the most ordinary query there is -- measured at
    19.8x on 1M rows -- while filtering those nulls globally is not merely safe
    there but exactly right, since a guarded conjunction matches no null row.

    The two are told apart by what the negation *wraps*.  A validity guard
    always negates a **simple leaf**: a comparison against the sentinel, or
    ``isnan`` for a NaN one.  A negation that changes whether a null can match
    wraps a boolean combination instead -- Kleene's ``~(unknown & false)`` is
    true, and that shape is always ``~`` of an ``&``/``|`` tree (see
    ``blosc2.ctable_nulls._NullPredicateRewriter._channels``).  Anything that is
    not recognizably a guard keeps the conservative answer.
    """
    try:
        tree = ast.parse(expression, mode="eval")
    except SyntaxError:
        return True
    for node in ast.walk(tree):
        if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or):
            return True
        if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
            return True
        if (
            isinstance(node, ast.UnaryOp)
            and isinstance(node.op, (ast.Not, ast.Invert))
            and not _negates_a_simple_leaf(node.operand)
        ):
            return True
    return False


def _negates_a_simple_leaf(operand: ast.AST) -> bool:
    """Whether a negation's *operand* is a leaf, so the negation cannot un-exclude a null.

    A comparison or a bare operand reference is a leaf; so is ``isnan(x)``,
    which is how a NaN sentinel spells its guard.  Other calls are deliberately
    **not** trusted: the string rewriter treats an unrecognized call as
    two-valued and emits no guard for it, so a null row can satisfy ``~f(a)``
    there and a global filter would drop a row the scan keeps.
    """
    if isinstance(operand, (ast.Compare, ast.Name, ast.Attribute)):
        return True
    return (
        isinstance(operand, ast.Call) and isinstance(operand.func, ast.Name) and operand.func.id == "isnan"
    )


class _DictRankWrapper:
    """Wrap a dictionary column's codes NDArray, translating codes to alphabetical ranks on read.

    Mirrors the NDArray interface enough for the index builder (dtype, shape, ndim,
    chunks, blocks, __getitem__).  The rank for code *c* is its position in an
    alphabetically-sorted dictionary, so sorting by rank == sorting by decoded string.
    Null codes map to a sentinel rank ``null_rank = len(dictionary)`` (largest, so
    nulls sort last).
    """

    def __init__(
        self,
        codes,
        code_to_rank: np.ndarray,
        null_rank: np.int32,
        null_code: int,
        nullable: bool,
        n_phys: int,
    ):
        self._codes = codes  # int32 NDArray
        self._code_to_rank = code_to_rank  # int32 array mapping code -> rank
        self._null_rank = null_rank
        self._null_code = null_code
        self._nullable = nullable
        self.dtype = np.dtype(np.int32)
        # The codes array carries capacity padding beyond the written rows; expose
        # only the physical extent so the index sidecars match n_rows (no padding →
        # the zero-permutation window read engages instead of falling back).  It has
        # to be the physical extent and not the live count: tombstoned rows keep
        # their positions, so live rows can sit past the live count.
        self.shape = (n_phys,)
        self.ndim = 1
        chunk0 = codes.chunks[0] if codes.chunks else n_phys
        block0 = codes.blocks[0] if codes.blocks else n_phys
        self.chunks = (min(chunk0, n_phys),)
        self.blocks = (min(block0, n_phys),)

    def __getitem__(self, key):
        codes_slice = np.asarray(self._codes[key], dtype=np.int32)
        ranks = self._code_to_rank[codes_slice]
        if self._nullable:
            ranks[codes_slice == self._null_code] = self._null_rank
        return ranks


class _CTableBuildProxy:
    """Minimal shim that lets the ``indexing`` module build sidecars for a
    CTable column without touching the column's own ``schunk.vlmeta``.

    Attributes mirror those required by the internal build functions:
    ``urlpath``, ``schunk``, ``shape``, ``ndim``, ``dtype``, ``chunks``,
    ``blocks``, and item access via ``__getitem__``.
    """

    def __init__(self, col_array: blosc2.NDArray, anchor_urlpath: str | None) -> None:
        self._col_array = col_array
        self.urlpath = anchor_urlpath  # controls sidecar placement
        self.schunk = _FakeSchunk()
        self.shape = col_array.shape
        self.ndim = col_array.ndim
        self.dtype = col_array.dtype
        self.chunks = col_array.chunks
        self.blocks = col_array.blocks

    def __getitem__(self, key):
        return self._col_array[key]


class _CTableIndexingMixin:
    # Cost-model constants for cross-column index refinement.
    # Calibrated from profiling with sparse-gather optimisations.
    #   _GATHER_COST_MS_PER_1K_ITEMS_PER_OP  ≈ ms to sparse-gather 1000 items from one operand column
    #   _SCAN_COST_MS_PER_1M_ROWS             ≈ ms to miniexpr-scan 1 million rows
    # If refinement cost exceeds scan cost, fall back to a full scan.
    _GATHER_COST_MS_PER_1K_ITEMS_PER_OP: float = 3.5
    _SCAN_COST_MS_PER_1M_ROWS: float = 4.3

    # Cost-model constants for the segment-summary (SUMMARY index) gate.
    # Surviving blocks are evaluated through miniexpr with a candidate-block
    # bitmap (non-candidate blocks are skipped inside the prefilter), so the
    # per-block overhead is small; what matters is that, once enough blocks
    # survive, evaluating them (plus iterating every chunk) costs more than a
    # plain sequential scan, and the planner falls back.
    #   _SEGMENT_SCAN_MS_PER_1M_ROWS_PER_OP ≈ ms to sequentially scan 1M rows of one column
    #   _SEGMENT_OVERHEAD_MS_PER_OP         ≈ fixed ms per surviving block per operand column
    _SEGMENT_SCAN_MS_PER_1M_ROWS_PER_OP: float = 1.6
    _SEGMENT_OVERHEAD_MS_PER_OP: float = 0.2

    @property
    def _root_table(self) -> CTable:
        """Return the root (non-view) table; *self* if not a view."""
        t = self
        while t.base is not None:
            t = t.base
        return t

    def _invalidate_index_catalog_cache(self) -> None:
        root = self._root_table
        root._cached_index_catalog = None
        root._cached_index_catalog_revision = None

    def _get_index_catalog(self) -> dict:
        root = self._root_table
        revision = root._storage.index_catalog_revision()
        catalog = getattr(root, "_cached_index_catalog", None)
        if catalog is None or getattr(root, "_cached_index_catalog_revision", None) != revision:
            catalog = root._storage.load_index_catalog()
            root._cached_index_catalog = catalog
            root._cached_index_catalog_revision = revision
        return catalog

    def _mark_all_indexes_stale(self) -> None:
        """Bump value_epoch and mark every catalog entry stale on the root table."""
        root = self._root_table
        root._storage.bump_value_epoch()
        catalog = root._get_index_catalog()
        if not catalog:
            return
        changed = False
        for desc in catalog.values():
            if not desc.get("stale", False):
                desc["stale"] = True
                changed = True
        if changed:
            root._storage.save_index_catalog(catalog)
            root._invalidate_index_catalog_cache()

    @staticmethod
    def _validate_index_descriptor(col_name: str, descriptor: dict) -> None:
        """Raise ValueError when an index catalog entry is malformed."""
        if not isinstance(descriptor, dict):
            raise ValueError(f"Malformed index metadata for column {col_name!r}: descriptor must be a dict.")
        token = descriptor.get("token")
        if not isinstance(token, str) or not token:
            raise ValueError(f"Malformed index metadata for column {col_name!r}: missing token.")
        kind = descriptor.get("kind")
        if kind not in {"summary", "bucket", "partial", "full", "opsi"}:
            raise ValueError(f"Malformed index metadata for column {col_name!r}: invalid kind {kind!r}.")
        if kind == "bucket" and not isinstance(descriptor.get("bucket"), dict):
            raise ValueError(f"Malformed index metadata for column {col_name!r}: missing bucket payload.")
        if kind == "partial" and not isinstance(descriptor.get("partial"), dict):
            raise ValueError(f"Malformed index metadata for column {col_name!r}: missing partial payload.")
        if kind == "full" and not isinstance(descriptor.get("full"), dict):
            raise ValueError(f"Malformed index metadata for column {col_name!r}: missing full payload.")

    def _drop_index_descriptor(self, col_name: str, descriptor: dict) -> None:
        """Delete sidecars/cache for a catalog descriptor without touching the column mapping."""
        from pathlib import Path

        from blosc2.indexing import (
            _IN_MEMORY_INDEXES,
            _PERSISTENT_INDEXES,
            _array_key,
            _clear_cached_data,
            _drop_descriptor_sidecars,
            _is_persistent_array,
        )

        token = descriptor["token"]
        col_arr = None
        with contextlib.suppress(Exception):
            col_arr = self._index_target_array(col_name, descriptor)

        if col_arr is not None:
            _clear_cached_data(col_arr, token)

        if col_arr is not None and _is_persistent_array(col_arr):
            arr_key = _array_key(col_arr)
            store = _PERSISTENT_INDEXES.get(arr_key)
            if store is not None:
                store["indexes"].pop(token, None)
        elif col_arr is not None:
            store = _IN_MEMORY_INDEXES.get(id(col_arr))
            if store is not None:
                store["indexes"].pop(token, None)

        _drop_descriptor_sidecars(descriptor)
        self._root_table._expr_index_arrays.pop(token, None)

        expr_values_path = descriptor.get("expr_values_path")
        if expr_values_path is not None:
            with contextlib.suppress(OSError):
                os.remove(expr_values_path)

        anchor = self._storage.index_anchor_path(col_name)
        if anchor is not None:
            proxy_key = ("persistent", str(Path(anchor).resolve()))
            _PERSISTENT_INDEXES.pop(proxy_key, None)
            with contextlib.suppress(OSError):
                os.rmdir(os.path.dirname(anchor))

    def _index_create_kwargs_from_descriptor(self, descriptor: dict) -> dict[str, Any]:
        """Return create_index kwargs that rebuild an existing descriptor."""
        build = "ooc" if bool(descriptor.get("ooc", False)) else "memory"
        kwargs = {
            "kind": descriptor["kind"],
            "optlevel": int(descriptor.get("optlevel", 5)),
            "name": descriptor.get("name") or None,
            "build": build,
            "cparams": descriptor.get("cparams"),
        }
        if descriptor.get("kind") == "full":
            kwargs["method"] = descriptor.get("full", {}).get("build_method", "global-sort")
        if descriptor.get("kind") == "opsi":
            kwargs["opsi_max_cycles"] = descriptor.get("opsi", {}).get("max_cycles")
        target = descriptor.get("target") or {}
        if target.get("source") == "expression":
            kwargs["expression"] = target.get("expression")
        return kwargs

    def _normalize_table_expression_target(
        self, expression: str, operands: dict | None = None
    ) -> tuple[dict, np.dtype]:
        """Normalize a same-table expression target and infer its dtype."""
        if operands is None:
            operands = self._cols
        try:
            ast.parse(expression, mode="eval")
        except SyntaxError as exc:
            raise ValueError("expression is not valid Python syntax") from exc

        owned_ids = {id(arr): name for name, arr in self._root_table._cols.items()}
        dependencies: list[str] = []
        valid = True

        class _Canonicalizer(ast.NodeTransformer):
            def visit_Name(self_inner, node: ast.Name) -> ast.AST:
                nonlocal valid
                operand = operands.get(node.id)
                if operand is None or not isinstance(operand, blosc2.NDArray):
                    return node
                cname = owned_ids.get(id(operand))
                if cname is None:
                    valid = False
                    return node
                dependencies.append(cname)
                return ast.copy_location(ast.Name(id=cname, ctx=node.ctx), node)

        normalized = _Canonicalizer().visit(
            ast.fix_missing_locations(ast.parse(expression, mode="eval")).body
        )
        if not valid or not dependencies:
            raise ValueError("expression indexes require operands from stored columns of the same table")
        dependencies = list(dict.fromkeys(dependencies))
        expression_key = ast.unparse(normalized)
        lazy = blosc2.lazyexpr(expression_key, {dep: self._root_table._cols[dep] for dep in dependencies})
        sample_stop = min(
            len(self._root_table._valid_rows), max(1, int(self._root_table._valid_rows.blocks[0]))
        )
        sample = lazy[:sample_stop]
        if isinstance(sample, blosc2.NDArray):
            sample = sample[:]
        sample = np.asarray(sample)
        dtype = np.dtype(sample.dtype)
        if sample.ndim != 1:
            raise ValueError("expression indexes require expressions returning a 1-D scalar stream")
        target = {
            "source": "expression",
            "expression": expression,
            "expression_key": expression_key,
            "dependencies": dependencies,
        }
        return target, dtype

    def _expression_index_values_path(self, token: str) -> str | None:
        anchor = self._storage.index_anchor_path(token)
        if anchor is None:
            return None
        return os.path.join(os.path.dirname(anchor), "values.b2nd")

    def _build_expression_values_array(self, target: dict, dtype: np.dtype, cparams=None) -> blosc2.NDArray:
        """Build a physical 1-D values array for a table expression target."""
        from blosc2.indexing import _target_token

        root = self._root_table
        capacity = len(root._valid_rows)
        chunks, blocks = compute_chunks_blocks((capacity,), dtype=dtype)
        urlpath = root._expression_index_values_path(_target_token(target))
        if urlpath is not None:
            os.makedirs(os.path.dirname(urlpath), exist_ok=True)
            arr = blosc2.zeros(
                (capacity,), dtype=dtype, urlpath=urlpath, mode="w", chunks=chunks, blocks=blocks
            )
        else:
            arr = blosc2.zeros((capacity,), dtype=dtype, chunks=chunks, blocks=blocks)
        lazy = blosc2.lazyexpr(
            target["expression_key"], {dep: root._cols[dep] for dep in target["dependencies"]}
        )
        step = int(root._valid_rows.chunks[0]) if root._valid_rows.chunks else 65536
        for start in range(0, capacity, step):
            stop = min(start + step, capacity)
            values = lazy[start:stop]
            if isinstance(values, blosc2.NDArray):
                values = values[:]
            arr[start:stop] = np.asarray(values, dtype=dtype)
        root._expr_index_arrays[_target_token(target)] = arr
        return arr

    def _index_target_array(self, lookup_key: str, descriptor: dict) -> blosc2.NDArray:
        """Return the physical array backing a column or expression index."""
        target = descriptor.get("target") or {}
        if target.get("source") != "expression":
            return self._root_table._cols[lookup_key]
        token = descriptor["token"]
        root = self._root_table
        arr = root._expr_index_arrays.get(token)
        if arr is not None:
            return arr
        path = descriptor.get("expr_values_path")
        if path is None:
            raise KeyError(f"No backing array found for expression index {token!r}.")
        arr = blosc2.open(path, mode="r" if root._read_only else "a")
        root._expr_index_arrays[token] = arr
        return arr

    def _resolve_index_catalog_entry(
        self, col_name: str | None = None, *, expression: str | None = None, name: str | None = None
    ) -> tuple[str, dict]:
        """Resolve an index catalog entry by column, expression, or label."""
        catalog = self._root_table._get_index_catalog()
        if col_name is not None and expression is not None:
            raise ValueError("col_name and expression are mutually exclusive")
        if col_name is not None:
            col_name = self._logical_to_physical_name(col_name)
            if col_name not in catalog:
                raise KeyError(f"No index found for column {col_name!r}.")
            return col_name, catalog[col_name]
        if expression is not None:
            from blosc2.indexing import _target_token

            target, _ = self._normalize_table_expression_target(expression)
            token = _target_token(target)
            if token not in catalog:
                raise KeyError(f"No index found for expression {expression!r}.")
            return token, catalog[token]
        if name is not None:
            matches = [(key, desc) for key, desc in catalog.items() if desc.get("name") == name]
            if not matches:
                raise KeyError(f"No index found with name {name!r}.")
            if len(matches) > 1:
                raise ValueError(f"Multiple indexes found with name {name!r}; specify a target explicitly.")
            return matches[0]
        raise TypeError("must specify col_name, expression, or name")

    def _index_validity_provider(self, col_name: str):
        """``(provider, null_aware)`` for *col_name*'s summary build.

        *provider* is a ``(values, start, stop) -> valid`` callable, or ``None``
        when every row is valid.  It is what makes an index null-aware: the
        segment-summary builder calls it per chunk and keeps null rows out of
        the per-segment extrema.  Without it a nullable column's summaries
        describe its fill (or its sentinel) as if it were data, which is why
        the ``min()``/``max()`` shortcut had to decline them.

        *null_aware* is the claim recorded in the descriptor, and it is true in
        one case where *provider* is ``None``: a mask column that has never
        held a null (no sidecar, per decision 9 of plans/mask-based-nulls.md).
        Its summaries agree with a null-skipping scan for free, so it should
        not pay the bail.
        """
        col = self._schema.columns_by_name.get(col_name)
        spec = col.spec if col is not None else None
        if spec is None:
            return None, False
        kind = kind_of_spec(spec)
        if kind == NULL_MASK:
            mask = self._null_mask(col_name)
            if mask is None:
                return None, True

            def _valid_from_sidecar(values, start, stop, _mask=mask):
                # One byte per row off the sidecar, and the values are not
                # consulted at all.
                valid = np.asarray(_mask[start:stop], dtype=bool)
                if valid.shape[0] < stop - start:
                    # Capacity the sidecar has not been grown over yet: those
                    # rows hold no data, and _grow() writes True over new tail
                    # rows, so "valid" is the consistent answer for them.
                    pad = np.ones(stop - start - valid.shape[0], dtype=bool)
                    valid = np.concatenate([valid, pad])
                return valid

            return _valid_from_sidecar, True
        if kind == NULL_SENTINEL:
            null_value = spec.null_value

            def _valid_from_sentinel(values, start, stop, _nv=null_value):
                return ~sentinel_mask(values, _nv)

            return _valid_from_sentinel, True
        return None, False

    def _build_index_persistent(
        self,
        col_name: str,
        col_arr: blosc2.NDArray,
        *,
        kind: str,
        optlevel: int,
        name_hint: str | None,
        build: str,
        tmpdir: str | None,
        cparams_obj,
        method: str | None = None,
        opsi_max_cycles: int | None = None,
        summary_levels: tuple[str, ...] | None = None,
        precomputed_summaries: dict | None = None,
        validity=None,
    ) -> dict:
        """Build index sidecar files for a persistent-table column; return the descriptor."""
        import tempfile
        from pathlib import Path

        from blosc2.indexing import (
            _PERSISTENT_INDEXES,
            _array_key,
            _build_bucket_descriptor,
            _build_bucket_descriptor_ooc,
            _build_descriptor,
            _build_full_descriptor,
            _build_full_descriptor_ooc,
            _build_levels_descriptor,
            _build_levels_descriptor_ooc,
            _build_opsi_descriptor,
            _build_partial_descriptor,
            _build_partial_descriptor_ooc,
            _copy_descriptor,
            _field_target_descriptor,
            _resolve_full_index_tmpdir,
            _resolve_ooc_mode,
            _target_token,
            _values_for_target,
        )

        anchor = self._storage.index_anchor_path(col_name)
        os.makedirs(os.path.dirname(anchor), exist_ok=True)
        proxy = _CTableBuildProxy(col_arr, anchor)
        proxy_key = _array_key(proxy)
        _PERSISTENT_INDEXES.pop(proxy_key, None)  # clear any stale cache entry

        target = _field_target_descriptor(None)
        token = _target_token(target)
        persistent = True
        dtype = col_arr.dtype
        use_ooc = _resolve_ooc_mode(kind, build)
        if opsi_max_cycles is None:
            opsi_max_cycles = max(1, optlevel if optlevel < 8 else optlevel * 2)

        if use_ooc:
            resolved_tmpdir = _resolve_full_index_tmpdir(proxy, tmpdir)
            levels = _build_levels_descriptor_ooc(
                proxy,
                target,
                token,
                kind,
                dtype,
                persistent,
                cparams_obj,
                summary_levels=summary_levels,
                precomputed_summaries=precomputed_summaries,
                validity=validity,
            )
            bucket = (
                _build_bucket_descriptor_ooc(
                    proxy, target, token, kind, dtype, optlevel, persistent, cparams_obj
                )
                if kind == "bucket"
                else None
            )
            partial = (
                _build_partial_descriptor_ooc(
                    proxy, target, token, kind, dtype, optlevel, persistent, cparams_obj
                )
                if kind == "partial"
                else None
            )
            full = None
            opsi = None
            if kind == "full":
                with tempfile.TemporaryDirectory(prefix="blosc2-index-ooc-", dir=resolved_tmpdir) as td:
                    full = _build_full_descriptor_ooc(
                        proxy, target, token, kind, dtype, persistent, Path(td), cparams_obj, optlevel
                    )
                    full["build_method"] = "global-sort"
            if kind == "opsi":
                opsi = _build_opsi_descriptor(
                    proxy, target, token, kind, dtype, persistent, cparams_obj, opsi_max_cycles, optlevel
                )
            descriptor = _build_descriptor(
                proxy,
                target,
                token,
                kind,
                optlevel,
                persistent,
                True,
                name_hint,
                dtype,
                levels,
                bucket,
                partial,
                full,
                cparams_obj,
                opsi,
                validity is not None,
            )
        else:
            values = _values_for_target(proxy, target)
            levels = _build_levels_descriptor(
                proxy,
                target,
                token,
                kind,
                dtype,
                values,
                persistent,
                cparams_obj,
                summary_levels=summary_levels,
                validity=validity,
            )
            bucket = (
                _build_bucket_descriptor(proxy, token, kind, values, optlevel, persistent, cparams_obj)
                if kind == "bucket"
                else None
            )
            partial = (
                _build_partial_descriptor(proxy, token, kind, values, optlevel, persistent, cparams_obj)
                if kind == "partial"
                else None
            )
            full = None
            opsi = None
            if kind == "full":
                full = _build_full_descriptor(proxy, token, kind, values, persistent, cparams_obj, optlevel)
                full["build_method"] = "global-sort"
            if kind == "opsi":
                opsi = _build_opsi_descriptor(
                    proxy, target, token, kind, dtype, persistent, cparams_obj, opsi_max_cycles, optlevel
                )
            descriptor = _build_descriptor(
                proxy,
                target,
                token,
                kind,
                optlevel,
                persistent,
                False,
                name_hint,
                dtype,
                levels,
                bucket,
                partial,
                full,
                cparams_obj,
                opsi,
                validity is not None,
            )

        result = _copy_descriptor(descriptor)
        _PERSISTENT_INDEXES.pop(proxy_key, None)  # evict proxy to avoid memory leak
        return result

    def create_index(  # noqa: C901
        self,
        col_name: str | None = None,
        *,
        field: str | None = None,
        expression: str | None = None,
        operands: dict | None = None,
        kind: blosc2.IndexKind | None = None,
        optlevel: int = 5,
        name: str | None = None,
        build: str = "auto",
        tmpdir: str | None = None,
        granularity: str | None = None,
        **kwargs,
    ) -> blosc2.Index:
        """Build and register an index for a stored column or table expression.

        For tables with **nested (dotted) column names**, pass the dotted leaf
        name directly::

            t.create_index("trip.begin.lon")
            t.where("trip.begin.lon > -87.7").nrows   # index is used automatically

        .. rubric:: Choosing an index kind

        ``BUCKET`` (the default) is the cheapest to build and store.
        It accelerates single‑column ``where`` queries and ``sort_by``
        reuse with approximate ordering derived from value
        quantization.  Sufficient for most workloads.

        ``FULL`` builds a globally sorted index that returns exact
        row positions for any range predicate.  It enables the
        **cross‑column refinement** planner path: when a multi‑column
        conjunction such as ``(tips > 100) & (km > 0) & (sec > 0)``
        indexes only the most selective column, the planner obtains
        compact exact positions from ``FULL`` and evaluates the
        remaining predicates on just those rows.  ``FULL`` is also
        ideal for ``sort_by`` reuse because it carries a complete
        sort order.

        ``PARTIAL`` builds a chunk‑local sorted payload with segment
        navigation.  It is cheaper to build than ``FULL`` (roughly
        half the raw storage) while still providing exact positions
        for cross‑column refinement.  Its exact positions are most
        compact for equality or narrow range queries; wide ranges
        may scan proportionally more candidate segments.

        ``OPSI`` is a specialised tier for approximate ordering;
        prefer ``FULL`` when a globally sorted ordered index is
        needed to accelerate ``sort_by``.

        ``SUMMARY`` stores only per‑segment min/max and is the
        lightest kind; it may still skip segments for broad range
        queries but cannot accelerate ``sort_by``.

        When *kind* is omitted it defaults to ``BUCKET``, except on ``utf8()``
        and ``dictionary()`` columns, which are indexed by alphabetical rank and
        only ever consulted through a ``FULL`` index — there the default is
        ``FULL``, and asking for any other kind raises ``ValueError`` rather
        than building an index that nothing would use.

        .. rubric:: SUMMARY granularity

        For ``kind=SUMMARY``, ``granularity`` controls the segment size of the
        min/max summaries: ``"block"`` (the default) summarizes at Blosc2 block
        granularity; ``"chunk"`` uses the coarser chunk granularity (fewer
        summary entries, cheaper to build, less selective pruning).
        ``granularity`` is only valid for ``kind=SUMMARY``.
        """
        if self.base is not None:
            raise ValueError("Cannot create an index on a view.")
        if col_name is not None and field is not None:
            raise ValueError("col_name and field are mutually exclusive")
        if expression is not None and (col_name is not None or field is not None):
            raise ValueError("column targets and expression are mutually exclusive")
        if operands is not None and expression is None:
            raise ValueError("operands can only be provided together with expression")
        col_name = field if field is not None else col_name
        if col_name is not None:
            col_name = self._logical_to_physical_name(col_name)

        from blosc2.indexing import (
            _IN_MEMORY_INDEXES,
            _copy_descriptor,
            _normalize_build_mode,
            _normalize_full_build_method,
            _normalize_index_cparams,
            _normalize_index_kind,
            _resolve_summary_levels,
            _target_token,
        )
        from blosc2.indexing import create_index as _ix_create_index

        cparams_obj = _normalize_index_cparams(kwargs.pop("cparams", None))
        method = kwargs.pop("method", None)
        opsi_max_cycles = kwargs.pop("opsi_max_cycles", None)
        precomputed_summaries = kwargs.pop("precomputed_summaries", None)
        if opsi_max_cycles is not None:
            opsi_max_cycles = max(1, int(opsi_max_cycles))
        if kwargs:
            raise TypeError(f"unexpected keyword argument(s): {', '.join(sorted(kwargs))}")

        # Rank-indexed flavours are only ever consulted through a FULL index, so
        # the BUCKET default would hand them an index nothing can use.  Default
        # per flavour, and remember whether the caller chose the kind: an
        # explicit non-FULL request is an error rather than a silent promotion.
        explicit_kind = kind is not None
        if kind is None:
            spec = None
            if col_name is not None:
                col_info = self._schema.columns_by_name.get(col_name)
                spec = col_info.spec if col_info is not None else None
            kind = (
                blosc2.IndexKind.FULL
                if isinstance(spec, (UTF8Spec, DictionarySpec))
                else blosc2.IndexKind.BUCKET
            )

        kind_str = _normalize_index_kind(kind)
        build_str = _normalize_build_mode(build)
        method_str = _normalize_full_build_method(method) if kind_str == "full" else None
        if method is not None and kind_str != "full":
            raise ValueError("method is only supported for kind=IndexKind.FULL")
        if granularity is not None and kind_str != "summary":
            raise ValueError("granularity is only supported for kind=IndexKind.SUMMARY")
        summary_levels = _resolve_summary_levels(granularity)
        catalog = self._get_index_catalog()

        if expression is not None:
            target, dtype = self._normalize_table_expression_target(expression, operands)
            token = _target_token(target)
            if token in catalog:
                raise ValueError(
                    f"Index already exists for expression {expression!r}. "
                    "Call rebuild_index() to replace it or drop_index() first."
                )
            expr_arr = self._build_expression_values_array(target, dtype, cparams=cparams_obj)
            _ix_create_index(
                expr_arr,
                kind=blosc2.IndexKind(kind_str),
                optlevel=optlevel,
                name=name,
                build=build,
                tmpdir=tmpdir,
                cparams=cparams_obj,
                method=method_str,
                opsi_max_cycles=opsi_max_cycles,
                summary_levels=summary_levels,
            )
            store = _IN_MEMORY_INDEXES.get(id(expr_arr))
            if store is None:
                from blosc2.indexing import _load_store

                store = _load_store(expr_arr)
            descriptor = _copy_descriptor(store["indexes"]["__self__"])
            descriptor["target"] = target
            descriptor["token"] = token
            descriptor["dtype"] = str(np.dtype(dtype))
            descriptor["expr_values_path"] = getattr(expr_arr, "urlpath", None)
            value_epoch, _ = self._storage.get_epoch_counters()
            descriptor["built_value_epoch"] = value_epoch
            catalog[token] = descriptor
            self._storage.save_index_catalog(catalog)
            self._invalidate_index_catalog_cache()
            return blosc2.Index._from_table(self, token, descriptor)

        if col_name is None:
            raise TypeError("must specify col_name/field or expression")
        if col_name in self._computed_cols:
            raise ValueError(
                f"Cannot create an index on computed column {col_name!r}: "
                "computed columns have no physical storage."
            )
        if col_name not in self._cols:
            raise KeyError(f"No column named {col_name!r}. Available: {self.col_names}")
        self._ensure_generated_column_not_stale(col_name)
        if col_name in catalog:
            raise ValueError(
                f"Index already exists for column {col_name!r}. "
                "Call rebuild_index() to replace it or drop_index() first."
            )

        col_arr = self._cols[col_name]
        if isinstance(self._schema.columns_by_name[col_name].spec, NDArraySpec):
            spec = self._schema.columns_by_name[col_name].spec
            raise ValueError(
                f"Cannot create an index on ndarray column {col_name!r} with per-row shape {spec.item_shape}. "
                "Materialize a scalar generated column first, e.g. embedding_norm or embedding_max."
            )
        if isinstance(self._schema.columns_by_name[col_name].spec, ListSpec):
            raise ValueError(f"Cannot create an index on list column {col_name!r} in V1.")
        if isinstance(
            self._schema.columns_by_name[col_name].spec, (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec)
        ):
            raise NotImplementedError(
                f"Cannot create an index on variable-length scalar column {col_name!r}: "
                "indexing for vlstring/vlbytes/struct/object columns is not supported yet."
            )
        # Rank-indexed flavours (utf8, dictionary) are queried through their own
        # literal->rank lookup rather than through plan_query, and both that path
        # and the ordering path require kind="full".  The other kinds build over
        # the ranks without error and are then never consulted, so refuse them
        # here rather than charge for an index nothing can use.
        rank_spec = self._schema.columns_by_name[col_name].spec
        if explicit_kind and kind_str != "full" and isinstance(rank_spec, (UTF8Spec, DictionarySpec)):
            flavour = "utf8" if isinstance(rank_spec, UTF8Spec) else "dictionary"
            raise ValueError(
                f"Column {col_name!r} is a {flavour} column, which is indexed by alphabetical rank; "
                f"only kind='full' consults that index, so kind={kind_str!r} would build but never "
                "be used. Use kind='full'."
            )
        # utf8 columns: index the alphabetical rank of each row's value.  There is
        # no stored code array to wrap lazily, so the ranks are materialized here
        # (int32, 4 B/row) and handed to the builder as an ordinary array.
        is_utf8 = isinstance(self._schema.columns_by_name[col_name].spec, UTF8Spec)
        utf8_rank_meta = None
        if is_utf8:
            # Span the physical extent, not the live count: delete() tombstones in
            # place, so live rows sit past _n_rows.  A utf8 column carries no
            # capacity padding (__len__ is persisted + pending), and this is the
            # same length _utf8_rank_index_stale() compares the meta against.
            n_phys = len(col_arr)
            ranks_arr, utf8_rank_meta, utf8_vocab = _utf8_rank_arrays(
                col_arr, n_phys, self[col_name].null_value, valid=self._null_mask(col_name)
            )
            col_arr = blosc2.asarray(ranks_arr)

        # Dictionary columns: index by alphabetical rank instead of insertion-order codes.
        is_dictionary = isinstance(self._schema.columns_by_name[col_name].spec, DictionarySpec)
        dict_rank_meta = None
        if is_dictionary:
            dict_col = col_arr
            # Physical extent again, but len(dict_col) is the slot *capacity*, so
            # take the live-data watermark instead: it covers every live row while
            # still excluding the trailing padding.
            n_phys = self._resolve_last_pos()
            dictionary = list(dict_col.dictionary)
            n_entries = len(dictionary)
            code_to_rank = _dict_code_to_rank(dictionary)
            null_code = dict_col.spec.null_code
            null_rank = np.int32(n_entries)
            # Hash for staleness detection.
            dict_hash = _dict_rank_hash(dictionary)
            dict_rank_meta = {"null_rank": int(null_rank), "dict_hash": dict_hash, "dict_len": n_entries}
            col_arr = _DictRankWrapper(
                dict_col.codes, code_to_rank, null_rank, null_code, dict_col.spec.nullable, n_phys
            )
        # utf8 and dictionary columns are indexed by *rank*, and a null already
        # has a rank of its own there (``null_rank``, sorting last), so their
        # summaries are null-aware by construction and there is nothing for a
        # validity channel to add -- nor would it line up, the ranks array being
        # a different array from the column's payload.
        validity, null_aware = (
            (None, False) if (is_utf8 or is_dictionary) else self._index_validity_provider(col_name)
        )

        is_persistent = self._storage.index_anchor_path(col_name) is not None

        if is_persistent:
            descriptor = self._build_index_persistent(
                col_name,
                col_arr,
                kind=kind_str,
                optlevel=optlevel,
                name_hint=name,
                build=build_str,
                tmpdir=tmpdir,
                cparams_obj=cparams_obj,
                method=method_str,
                opsi_max_cycles=opsi_max_cycles,
                summary_levels=summary_levels,
                precomputed_summaries=precomputed_summaries if kind_str == "summary" else None,
                validity=validity,
            )
        else:
            # In-memory path: materialise ranks as a proper NDArray (small tables only).
            if is_dictionary:
                codes = np.asarray(dict_col.codes[:n_phys], dtype=np.int32)
                ranks_arr = code_to_rank[codes]
                if dict_col.spec.nullable:
                    ranks_arr[codes == null_code] = null_rank
                col_arr = blosc2.asarray(ranks_arr)
            _ix_create_index(
                col_arr,
                field=None,
                kind=blosc2.IndexKind(kind_str),
                optlevel=optlevel,
                name=name,
                build=build,
                tmpdir=tmpdir,
                cparams=cparams_obj,
                method=method_str,
                opsi_max_cycles=opsi_max_cycles,
                summary_levels=summary_levels,
                precomputed_summaries=precomputed_summaries if kind_str == "summary" else None,
                validity=validity,
            )
            store = _IN_MEMORY_INDEXES[id(col_arr)]
            descriptor = _copy_descriptor(store["indexes"]["__self__"])
        if dict_rank_meta is not None or utf8_rank_meta is not None:
            full = descriptor.setdefault("full", {})
            if full is None:
                full = descriptor["full"] = {}
            if dict_rank_meta is not None:
                full["dict_rank"] = dict_rank_meta
            else:
                _persist_utf8_vocab(full, utf8_rank_meta, utf8_vocab)
                full["utf8_rank"] = utf8_rank_meta

        # A null-free mask column builds with no validity provider (there is
        # nothing to exclude) yet its summaries are null-aware all the same.
        descriptor["null_aware"] = bool(descriptor.get("null_aware") or null_aware)

        value_epoch, _ = self._storage.get_epoch_counters()
        descriptor["built_value_epoch"] = value_epoch

        if is_persistent:
            # Use column name as token so sibling columns in compact stores get
            # distinct keys in the shared _PERSISTENT_INDEXES store (all columns
            # share the same urlpath yet must not collide on "__self__").  File
            # paths were built with token="__self__" (omitted from the filename)
            # and remain unchanged; only the in-memory key is affected.
            # In-memory CTables already have unique _IN_MEMORY_INDEXES entries
            # per column (keyed by id(array)), so no token change is needed.
            descriptor["token"] = col_name

        catalog = self._get_index_catalog()
        catalog[col_name] = descriptor
        self._storage.save_index_catalog(catalog)
        self._invalidate_index_catalog_cache()
        return blosc2.Index._from_table(self, col_name, descriptor)

    def drop_index(
        self, col_name: str | None = None, *, expression: str | None = None, name: str | None = None
    ) -> None:
        """Remove an index and delete any sidecar files."""
        if self.base is not None:
            raise ValueError("Cannot drop an index from a view.")

        lookup_key, descriptor = self._resolve_index_catalog_entry(
            col_name, expression=expression, name=name
        )
        catalog = self._get_index_catalog()
        catalog.pop(lookup_key, None)
        self._validate_index_descriptor(lookup_key, descriptor)
        self._drop_index_descriptor(lookup_key, descriptor)
        self._storage.save_index_catalog(catalog)
        self._invalidate_index_catalog_cache()

    def rebuild_index(
        self, col_name: str | None = None, *, expression: str | None = None, name: str | None = None
    ) -> blosc2.Index:
        """Drop and recreate an index with the same parameters."""
        if self.base is not None:
            raise ValueError("Cannot rebuild an index on a view.")

        lookup_key, old_desc = self._resolve_index_catalog_entry(col_name, expression=expression, name=name)
        self._validate_index_descriptor(lookup_key, old_desc)
        create_kwargs = self._index_create_kwargs_from_descriptor(old_desc)

        self.drop_index(col_name, expression=expression, name=name)
        if "expression" in create_kwargs:
            return self.create_index(expression=create_kwargs.pop("expression"), **create_kwargs)
        return self.create_index(lookup_key, **create_kwargs)

    def compact_index(
        self, col_name: str | None = None, *, expression: str | None = None, name: str | None = None
    ) -> blosc2.Index:
        """Compact an index, merging any incremental append runs."""
        if self.base is not None:
            raise ValueError("Cannot compact an index on a view.")

        from blosc2.indexing import (
            _IN_MEMORY_INDEXES,
            _PERSISTENT_INDEXES,
            _array_key,
            _copy_descriptor,
            _default_index_store,
            _is_persistent_array,
        )
        from blosc2.indexing import compact_index as _ix_compact_index

        lookup_key, descriptor = self._resolve_index_catalog_entry(
            col_name, expression=expression, name=name
        )
        col_arr = self._index_target_array(lookup_key, descriptor)
        catalog = self._get_index_catalog()

        if _is_persistent_array(col_arr):
            anchor = self._storage.index_anchor_path(lookup_key)
            proxy = _CTableBuildProxy(col_arr, anchor)
            proxy_key = _array_key(proxy)
            store = _default_index_store()
            store["indexes"][descriptor["token"]] = descriptor
            _PERSISTENT_INDEXES[proxy_key] = store
            try:
                _ix_compact_index(proxy)
                updated_store = _PERSISTENT_INDEXES.get(proxy_key) or store
                updated_desc = _copy_descriptor(updated_store["indexes"][descriptor["token"]])
            finally:
                _PERSISTENT_INDEXES.pop(proxy_key, None)
            updated_desc["built_value_epoch"] = descriptor.get("built_value_epoch", 0)
            catalog[lookup_key] = updated_desc
            self._storage.save_index_catalog(catalog)
            self._invalidate_index_catalog_cache()
            return blosc2.Index._from_table(self, lookup_key, updated_desc)
        else:
            _ix_compact_index(col_arr)
            store = _IN_MEMORY_INDEXES.get(id(col_arr))
            if store:
                token = descriptor["token"]
                updated_desc = _copy_descriptor(store["indexes"].get(token, descriptor))
                updated_desc["built_value_epoch"] = descriptor.get("built_value_epoch", 0)
                catalog[lookup_key] = updated_desc
                self._storage.save_index_catalog(catalog)
                self._invalidate_index_catalog_cache()
                return blosc2.Index._from_table(self, lookup_key, updated_desc)
            return blosc2.Index._from_table(self, lookup_key, descriptor)

    def index(
        self, col_name: str | None = None, *, expression: str | None = None, name: str | None = None
    ) -> blosc2.Index:
        """Return the index handle for a stored-column or expression target."""
        lookup_key, descriptor = self._resolve_index_catalog_entry(
            col_name, expression=expression, name=name
        )
        return blosc2.Index._from_table(self, lookup_key, descriptor)

    @property
    def indexes(self) -> list[blosc2.Index]:
        """Return a list of :class:`blosc2.Index` handles for all active indexes."""
        catalog = self._root_table._get_index_catalog()
        return [blosc2.Index._from_table(self, col_name, desc) for col_name, desc in catalog.items()]

    def _rewrite_expression_query_for_index(
        self, expression: str, operands: dict, target: dict
    ) -> str | None:
        """Rewrite matching table-expression subtrees to ``_where_x`` for planning."""
        try:
            tree = ast.parse(expression, mode="eval")
        except SyntaxError:
            return None

        class _Rewriter(ast.NodeTransformer):
            def __init__(self, outer):
                self.outer = outer
                self.changed = False

            def generic_visit(self, node):
                normalized = None
                with contextlib.suppress(Exception):
                    normalized, _ = self.outer._normalize_table_expression_target(
                        ast.unparse(node), operands
                    )
                if normalized is not None and normalized.get("expression_key") == target.get(
                    "expression_key"
                ):
                    self.changed = True
                    return ast.copy_location(ast.Name(id="_where_x", ctx=ast.Load()), node)
                return super().generic_visit(node)

        rewriter = _Rewriter(self)
        new_body = rewriter.visit(tree.body)
        if not rewriter.changed:
            return None
        return ast.unparse(new_body)

    def _try_expression_index_where(self, expr_result: blosc2.LazyExpr, catalog: dict) -> np.ndarray | None:
        """Attempt to resolve *expr_result* via a direct table expression index."""
        from blosc2.indexing import evaluate_bucket_query, evaluate_segment_query, plan_query

        expression = expr_result.expression
        operands = dict(expr_result.operands)
        for lookup_key, descriptor in catalog.items():
            target = descriptor.get("target") or {}
            if target.get("source") != "expression" or descriptor.get("stale", False):
                continue
            rewritten = self._rewrite_expression_query_for_index(expression, operands, target)
            if rewritten is None:
                continue
            expr_arr = self._index_target_array(lookup_key, descriptor)
            where_dict = {"_where_x": expr_arr}
            merged_operands = {"_where_x": expr_arr}
            plan = plan_query(rewritten, merged_operands, where_dict)
            if not plan.usable:
                continue
            if plan.exact_positions is not None:
                return np.asarray(plan.exact_positions, dtype=np.int64)
            if plan.bucket_masks is not None:
                _, positions = evaluate_bucket_query(
                    rewritten, merged_operands, {}, where_dict, plan, return_positions=True
                )
                return np.asarray(positions, dtype=np.int64)
            if plan.candidate_units is not None and plan.segment_len is not None:
                _, positions = evaluate_segment_query(
                    rewritten, merged_operands, {}, where_dict, plan, return_positions=True
                )
                return np.asarray(positions, dtype=np.int64)
        return None

    @staticmethod
    def _evaluate_expression_at(expr_result, candidates, *, prefetched: dict | None = None):
        """Evaluate *expr_result* on the operand rows at *candidates*.

        Returns a boolean ``numpy.ndarray`` the same length as *candidates*,
        or ``None`` if evaluation fails.

        Parameters
        ----------
        prefetched:
            Optional dict mapping operand variable names to already-gathered
            NumPy arrays.  When provided, those operands are reused instead of
            re-read from storage.
        """
        try:
            operands = {}
            for var_name, arr in expr_result.operands.items():
                if prefetched is not None and var_name in prefetched:
                    sliced = prefetched[var_name]
                else:
                    sliced = arr[candidates]
                    if hasattr(sliced, "__array__"):
                        sliced = np.asarray(sliced)
                operands[var_name] = sliced
            return blosc2.evaluate(expr_result.expression, operands)
        except Exception:
            return None

    @staticmethod
    def _find_indexed_columns(root_cols, catalog, operands):
        """Return live indexed columns referenced by *operands* in expression order.

        Avoid iterating over ``root_cols.items()`` here: for lazy persistent tables
        that would open every column just to find the indexed operands.
        """
        indexed = []
        seen = set()
        indexed_arrays = {}
        # Any column referenced by *operands* was already opened to build the
        # predicate expression, so it is present in the column cache.  A still
        # un-materialized indexed column therefore cannot be an operand: skip it
        # instead of opening it.  Without this, a predicate on a few columns
        # would eagerly open *every* indexed column (e.g. all 11 SUMMARY indexes
        # on a wide persistent table) — a pure cold-start cost.  Falling back to
        # the old behaviour for non-dict mappings keeps correctness; at worst a
        # genuinely-unloaded operand would miss the index and full-scan.
        cache_is_dict = isinstance(root_cols, dict)
        for col_name, descriptor in catalog.items():
            if col_name not in root_cols:
                continue
            if cache_is_dict and not dict.__contains__(root_cols, col_name):
                continue
            indexed_arrays[col_name] = (root_cols[col_name], descriptor)

        for operand in operands.values():
            if not isinstance(operand, blosc2.NDArray):
                continue
            for col_name, (col_arr, descriptor) in indexed_arrays.items():
                if col_name in seen or col_arr is not operand:
                    continue
                _CTableIndexingMixin._validate_index_descriptor(col_name, descriptor)
                if descriptor.get("stale", False):
                    continue
                indexed.append((col_name, col_arr, descriptor))
                seen.add(col_name)
        return indexed

    @staticmethod
    def _project_units_to_block_bitmap(primary_col_arr, candidate_units, segment_len) -> np.ndarray | None:
        """Project surviving flat segments onto a per-block candidate bitmap in
        *miniexpr* block coordinates (1-D only).

        Returns a ``uint8`` array with one byte per global miniexpr block
        (``nchunk * blocks_per_chunk + nblock``): 1 = the block overlaps a
        surviving segment and must be evaluated, 0 = it can be skipped.  A block
        is marked conservatively (any overlap with a survivor), which is always
        safe because the prefilter re-evaluates the exact predicate per row.
        """
        chunks = getattr(primary_col_arr, "chunks", None)
        blocks = getattr(primary_col_arr, "blocks", None)
        shape = getattr(primary_col_arr, "shape", None)
        if not chunks or not blocks or not shape or len(shape) != 1:
            return None
        chunk_len = int(chunks[0])
        block_len = int(blocks[0])
        nrows = int(shape[0])
        seg_len = int(segment_len)
        if chunk_len <= 0 or block_len <= 0 or nrows <= 0 or seg_len <= 0:
            return None
        cu = np.asarray(candidate_units, dtype=bool)
        nseg = len(cu)
        blocks_per_chunk = -(-chunk_len // block_len)  # ceil
        nchunks = -(-nrows // chunk_len)
        nblocks = nchunks * blocks_per_chunk
        g = np.arange(nblocks, dtype=np.int64)
        nchunk = g // blocks_per_chunk
        nblock = g % blocks_per_chunk
        lo = nchunk * chunk_len + nblock * block_len
        chunk_end = np.minimum((nchunk + 1) * chunk_len, nrows)
        hi = np.minimum(lo + block_len, chunk_end)
        has_rows = lo < chunk_end  # padding-only blocks have no real rows
        # Segments overlapping [lo, hi); test via a cumulative-sum range query.
        s_lo = np.clip(lo // seg_len, 0, nseg)
        s_hi = np.clip(np.where(has_rows, (hi - 1) // seg_len, lo // seg_len) + 1, 0, nseg)
        csum = np.concatenate(([0], np.cumsum(cu.astype(np.int64))))
        any_survivor = (csum[s_hi] - csum[s_lo]) > 0
        return (has_rows & any_survivor).astype(np.uint8)

    @staticmethod
    def _summary_candidate_block_bitmap(primary_col_arr, plan) -> np.ndarray | None:
        """Single-column candidate-block bitmap from a SUMMARY plan."""
        return _CTableIndexingMixin._project_units_to_block_bitmap(
            primary_col_arr, plan.candidate_units, plan.segment_len
        )

    @staticmethod
    def _flatten_conjunction(expression: str):
        """Return the list of ``ast.Compare`` leaves of a pure conjunction
        (``&`` / ``and`` of comparisons), or ``None`` if the expression is not a
        pure conjunction of comparisons (e.g. it contains ``|``, ``~``, or a
        bare boolean operand)."""
        import ast

        try:
            root = ast.parse(expression, mode="eval").body
        except (SyntaxError, ValueError):
            return None
        out = []
        stack = [root]
        while stack:
            node = stack.pop()
            if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.And):
                stack.extend(node.values)
            elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitAnd):
                stack.extend((node.left, node.right))
            elif isinstance(node, ast.Compare):
                out.append(node)
            else:
                return None  # not a pure conjunction of comparisons
        return out

    def _combined_summary_block_bitmap(self, expr_result, indexed_columns, primary_col_arr):
        """Combine per-column SUMMARY block masks for a conjunction.

        For ``(a > x) & (b > y) & (c < z)`` where several of ``a``/``b``/``c``
        carry SUMMARY indexes, each column's summaries yield a candidate-block
        mask (in the shared miniexpr-block grid); ANDing them prunes a block
        unless *every* indexed predicate could match in it.  Returns the combined
        ``uint8`` bitmap, or ``None`` when the expression is not a pure
        conjunction or no indexed column contributes a usable range predicate.
        """
        from blosc2.indexing import (
            _candidate_units_from_summary_handle,
            _clear_cached_data,
            _finest_level,
            _open_level_summary_handle,
            _target_from_compare,
        )

        compares = self._flatten_conjunction(expr_result.expression)
        if compares is None:
            return None
        operands = expr_result.operands
        desc_by_id = {id(arr): desc for _name, arr, desc in indexed_columns}
        per_col: dict[int, np.ndarray] = {}
        for node in compares:
            target = _target_from_compare(node, operands)
            if target is None:
                continue
            base, _target_info, op, value = target
            desc = desc_by_id.get(id(base))
            if desc is None or desc.get("kind") != "summary":
                continue  # non-indexed predicate: contributes no pruning (sound for AND)
            level = _finest_level(desc)
            try:
                # In compact stores every column shares one urlpath, so the
                # sidecar-handle cache key collides across columns; clear it so
                # each column opens *its own* summary sidecar, not a sibling's.
                _clear_cached_data(base, desc["token"])
                summaries = _open_level_summary_handle(base, desc, level)
                cu = _candidate_units_from_summary_handle(summaries, op, value, np.dtype(desc["dtype"]))
            except (RuntimeError, ValueError, TypeError, KeyError):
                continue
            seg_len = desc["levels"][level]["segment_len"]
            bm = self._project_units_to_block_bitmap(primary_col_arr, cu, seg_len)
            if bm is None:
                continue
            # Multiple predicates on the same column AND together (e.g. a>1 & a<5).
            per_col[id(base)] = bm if id(base) not in per_col else (per_col[id(base)] & bm)
        if not per_col:
            return None
        combined = None
        for bm in per_col.values():
            combined = bm if combined is None else (combined & bm)
        return combined

    def _block_pruned_positions(self, expr_result, bitmap, primary_col_arr):
        """Evaluate the predicate through miniexpr with *bitmap* and return the
        matching physical positions.

        Chunks with no candidate block are skipped inside fast_eval
        (``_prune_chunks``), and positions are extracted by **streaming** the
        candidate chunks one at a time (``decompress_chunk`` → ``flatnonzero`` →
        offset).  This never materializes the full boolean mask as one big numpy
        array — only one chunk (~chunk_len bytes) is uncompressed at a time —
        and is faster than a bulk ``mask[:]`` + ``flatnonzero`` even when every
        chunk is a candidate.  Returns the positions array, or ``None`` if the
        bitmap path is unavailable.
        """
        chunks = getattr(primary_col_arr, "chunks", None)
        blocks = getattr(primary_col_arr, "blocks", None)
        shape = getattr(primary_col_arr, "shape", None)
        chunk_mask = None
        if chunks and blocks and shape and len(shape) == 1:
            chunk_len = int(chunks[0])
            block_len = int(blocks[0])
            bpc = -(-chunk_len // block_len)
            if block_len > 0 and len(bitmap) % bpc == 0:
                chunk_mask = bitmap.reshape(len(bitmap) // bpc, bpc).any(axis=1)
        try:
            if chunk_mask is None:
                # Geometry unknown (e.g. non-1-D): fall back to a bulk mask read.
                mask = expr_result.compute(_candidate_blocks=bitmap)
                mask_arr = mask[:] if isinstance(mask, blosc2.NDArray) else np.asarray(mask)
                if mask_arr.dtype != np.bool_:
                    return None
                return np.flatnonzero(mask_arr).astype(np.int64)

            nrows = int(shape[0])
            mask = expr_result.compute(_candidate_blocks=bitmap, _prune_chunks=True)
            if not isinstance(mask, blosc2.NDArray) or mask.dtype != np.bool_:
                return None
            if mask.schunk.nchunks != len(chunk_mask):
                return None  # unexpected geometry; bail rather than read garbage
            parts = []
            for chunk_idx in np.flatnonzero(chunk_mask):
                chunk_idx = int(chunk_idx)
                lo = chunk_idx * chunk_len
                vlen = min(chunk_len, nrows - lo)  # trim block-alignment padding
                sub = np.frombuffer(mask.schunk.decompress_chunk(chunk_idx), dtype=np.bool_)[:vlen]
                rel = np.flatnonzero(sub)
                if len(rel):
                    parts.append(rel.astype(np.int64) + lo)
            return np.concatenate(parts) if parts else np.empty(0, dtype=np.int64)
        except Exception:
            return None

    def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None:  # noqa: C901
        """Attempt to resolve *expr_result* via a column index.

        Returns a 1-D int64 array of physical row positions that satisfy the
        predicate, or ``None`` if no usable index was found (caller falls back
        to a full scan).
        """
        from blosc2.indexing import (
            _IN_MEMORY_INDEXES,
            _PERSISTENT_INDEXES,
            _array_key,
            _default_index_store,
            _is_persistent_array,
            evaluate_bucket_query,
            evaluate_segment_query,
            plan_query,
        )

        root = self._root_table
        catalog = root._get_index_catalog()
        if not catalog:
            return None

        positions = self._try_expression_index_where(expr_result, catalog)
        if positions is not None:
            return positions

        expression = expr_result.expression
        operands = dict(expr_result.operands)

        indexed_columns = self._find_indexed_columns(root._cols, catalog, operands)
        if not indexed_columns:
            return None

        primary_col_name, primary_col_arr, _ = indexed_columns[0]
        # Null exclusion still happens here, even though every predicate that
        # reaches this point is null-aware (CTable._rewrite_null_predicates for
        # the string form, Column._null_aware_compare for the operator one).
        # An ordered index does not *evaluate* the predicate: it answers
        # ``a > 90`` by taking a range of the sorted column, and the sentinel
        # lives in that range whether or not it would satisfy the comparison --
        # a NaN sentinel sorts last, so every NaN row comes back for any
        # ``>`` query.  The expression being correct therefore does not make the
        # index result correct, and these positions must still be filtered.
        # Mask storage is in the same position as a sentinel here, with the fill
        # playing the sentinel's part: the ordered range that answers ``f > 0.5``
        # contains the NaN fill of every null row, so a mask column's nulls come
        # back too unless they are filtered out.  A mask column with no sidecar
        # has no nulls at all, so there is nothing for it to contribute.
        null_kinds = {}
        for name, _arr, _descriptor in indexed_columns:
            kind = kind_of_spec(root._schema.columns_by_name[name].spec)
            if kind == NULL_SENTINEL or (kind == NULL_MASK and root._null_mask(name) is not None):
                null_kinds[name] = kind
        nullable_indexed = list(null_kinds)
        # Only non-NaN sentinels need the *positions* fall-back below; for a NaN
        # sentinel the mask-direct path can stay, because that path evaluates the
        # predicate through miniexpr rather than reading an ordered range.  A
        # mask column is in the same position: whatever its fill, the predicate
        # miniexpr evaluates carries the sidecar guard that
        # ``CTable._rewrite_null_predicates`` conjoined onto every leaf the fill
        # could have satisfied.
        nullable_needs_exclude = [
            name
            for name, kind in null_kinds.items()
            if kind == NULL_SENTINEL
            and not is_nan_sentinel(root._schema.columns_by_name[name].spec.null_value)
        ]

        # Global null post-filtering is not correct for an OR expression -- it
        # would drop a row that is null in one column but matches the other
        # branch -- nor for a negation, where ``~(unknown & false)`` is true and
        # the row it would drop qualifies.  Which paths that rules out depends
        # on *how* each one answers.
        #
        # The segment/candidate-unit path evaluates the predicate through
        # miniexpr over the surviving blocks, and the predicate is null-aware
        # per leaf by the time it gets here (CTable._rewrite_null_predicates for
        # the string form, Column._null_aware_compare for the operator one), so
        # its result is already exact and the post-filter is not merely
        # incorrect for those shapes but unnecessary.  That path therefore
        # serves them, with the filter skipped.
        #
        # The exact-position paths (FULL/PARTIAL/BUCKET) answer by taking an
        # ordered *range* of the sorted column and never evaluate anything, so
        # the post-filter is load-bearing there and such an expression over a
        # nullable indexed column still falls back to the scan -- which is
        # itself null-aware, and so returns the right answer.  Those bails are
        # belt-and-braces today: the planner declines an exact plan for an
        # expression containing an OR or a negation, so such a plan cannot reach
        # them.  They are what stops a planner that learns to build one from
        # silently reintroducing the bug.
        skip_null_filter = bool(nullable_indexed) and _expression_defeats_global_null_filter(
            expr_result.expression
        )

        # Inject every usable table-owned descriptor so plan_query can combine them.
        # In .b2z read mode all columns share the same urlpath, so _array_key()
        # returns the same key for every column — causing _SIDECAR_HANDLE_CACHE
        # collisions across queries.  Clear stale handles before each injection so
        # the upcoming query always loads the correct sidecar for this column.
        from blosc2.indexing import _clear_cached_data, _register_descriptor_owner

        # Build column-name → array mapping so the planner can resolve the
        # correct per-column token instead of the generic "__self__".
        array_to_col = {}
        for _col_name, col_arr, descriptor in indexed_columns:
            array_to_col[id(col_arr)] = _col_name
            arr_key = _array_key(col_arr)
            if _is_persistent_array(col_arr):
                store = _PERSISTENT_INDEXES.get(arr_key) or _default_index_store()
                if store["indexes"].get(_col_name) is not descriptor:
                    _clear_cached_data(col_arr, _col_name)
                store["indexes"][_col_name] = descriptor
                _PERSISTENT_INDEXES[arr_key] = store
            else:
                store = _IN_MEMORY_INDEXES.get(id(col_arr)) or _default_index_store()
                store["indexes"][_col_name] = descriptor
                _IN_MEMORY_INDEXES[id(col_arr)] = store
            # Record the owning array so a sibling column sharing this urlpath
            # (and, after column alignment, the same shape/chunks) cannot match
            # this descriptor in _descriptor_for_target.
            _register_descriptor_owner(col_arr, _col_name)

        where_dict = {"_where_x": primary_col_arr}
        merged_operands = {**operands, "_where_x": primary_col_arr}

        plan = plan_query(expression, merged_operands, where_dict, array_to_col=array_to_col)
        if not plan.usable:
            return None

        def _exclude_null_positions(positions):
            positions = np.asarray(positions, dtype=np.int64)
            for name, kind in null_kinds.items():
                if positions.size == 0:
                    break
                if kind == NULL_MASK:
                    # One byte per candidate off the sidecar, and the values are
                    # not read at all -- the cheaper half of what masks buy.
                    keep = np.asarray(root._null_mask(name)[positions], dtype=bool)
                else:
                    nv = root._schema.columns_by_name[name].spec.null_value
                    keep = ~sentinel_mask(root._cols[name][positions], nv)
                positions = positions[keep]
            return positions

        if plan.exact_positions is not None:
            if skip_null_filter:
                return None  # ordered range, no per-leaf null handling: see above
            return _exclude_null_positions(plan.exact_positions)

        if plan.partial_exact_positions is not None:
            if skip_null_filter:
                return None  # ditto, and its refinement filter is global too
            # Cross-column refinement: the FULL index on one column gave us
            # exact positions, but the expression has additional predicates on
            # other columns.  Refinement reads every operand column at those
            # candidate positions using sparse/fancy indexing.  For compressed
            # columns this can touch many chunks and be slower than the regular
            # sequential miniexpr scan, which is very fast for simple predicates.
            # Use a cost model to compare refinement vs full scan.
            candidates = np.asarray(plan.partial_exact_positions, dtype=np.int64)
            n_candidates = len(candidates)
            n_operands = len(expr_result.operands)
            target_len = len(root._valid_rows)

            estimated_refine_ms = (
                (n_candidates / 1000.0) * self._GATHER_COST_MS_PER_1K_ITEMS_PER_OP * n_operands
            )
            estimated_scan_ms = (target_len / 1_000_000.0) * self._SCAN_COST_MS_PER_1M_ROWS
            if estimated_refine_ms > estimated_scan_ms:
                return None

            # Read the primary column once and reuse for both null filtering
            # and refinement, avoiding a second sparse gather later.
            primary_op_name = next(
                (vn for vn, va in expr_result.operands.items() if va is primary_col_arr), None
            )
            prefetched = None
            if nullable_indexed and primary_op_name is not None:
                raw = primary_col_arr[candidates]
                raw = np.asarray(raw) if hasattr(raw, "__array__") else raw
                # One combined keep-mask over the full candidate set, so the
                # prefetched primary values stay aligned with the positions
                # however many columns contribute nulls.  Narrowing the
                # positions column by column while trimming ``raw`` only for the
                # primary would silently misalign the two.
                keep = np.ones(len(candidates), dtype=bool)
                for name, kind in null_kinds.items():
                    if kind == NULL_MASK:
                        keep &= np.asarray(root._null_mask(name)[candidates], dtype=bool)
                        continue
                    nv = root._schema.columns_by_name[name].spec.null_value
                    values = raw if name == primary_col_name else root._cols[name][candidates]
                    keep &= ~sentinel_mask(values, nv)
                candidates = candidates[keep]
                # Reuse the primary read for refinement, saving a second sparse
                # gather -- but only when it survived as an aligned array.
                prefetched = {primary_op_name: raw[keep]} if isinstance(raw, np.ndarray) else None
            else:
                candidates = _exclude_null_positions(candidates)

            restricted = self._evaluate_expression_at(expr_result, candidates, prefetched=prefetched)
            if restricted is not None and restricted.dtype == np.bool_:
                refined = candidates[np.asarray(restricted, dtype=bool)]
                return _exclude_null_positions(refined)
            # Fall through to full scan if refinement fails

        if plan.bucket_masks is not None:
            if skip_null_filter:
                return None  # bucket masks are precomputed, not evaluated
            # When bucket pruning covers all units (100 % of chunks are
            # candidates), the per‑chunk evaluation overhead outweighs the
            # benefit over a plain scan.  Fall back to the scan path.
            if plan.total_units > 0 and plan.selected_units >= plan.total_units:
                return None
            _, positions = evaluate_bucket_query(
                expression, merged_operands, {}, where_dict, plan, return_positions=True
            )
            return _exclude_null_positions(positions)

        if plan.candidate_units is not None and plan.segment_len is not None:
            # Cheap pre-gate on the primary column's pruning (the plan is already
            # computed).  When the primary alone cannot prune enough, a plain scan
            # is cheaper, so fall back before doing any extra (per-column) work.
            n_ops = max(1, len(expr_result.operands))
            segment_len = int(plan.segment_len)
            selected = int(plan.selected_units)
            total_rows = int(plan.total_units) * segment_len
            selected_rows = selected * segment_len
            scan_ms = (total_rows / 1e6) * self._SEGMENT_SCAN_MS_PER_1M_ROWS_PER_OP * n_ops
            index_ms = (
                selected * n_ops * self._SEGMENT_OVERHEAD_MS_PER_OP
                + (selected_rows / 1e6) * self._SEGMENT_SCAN_MS_PER_1M_ROWS_PER_OP * n_ops
            )
            if selected > 0 and index_ms >= scan_ms:
                return None
            # Preferred path: evaluate the predicate through miniexpr with a
            # candidate-block bitmap, so non-surviving blocks are skipped inside
            # the fast_eval/miniexpr engine (no per-segment Python loop).  For a
            # conjunction, every column with a SUMMARY index contributes a block
            # mask and they are ANDed, pruning more than the primary alone.  The
            # masked result equals a full scan's (pruned blocks have no matches),
            # so positions are exact.  Falls back to the per-segment query if the
            # bitmap path is unavailable.
            bitmap = self._combined_summary_block_bitmap(expr_result, indexed_columns, primary_col_arr)
            if bitmap is None:
                bitmap = self._summary_candidate_block_bitmap(primary_col_arr, plan)
            if bitmap is not None:
                # Mask-direct: SUMMARY evaluates the predicate through miniexpr
                # with the candidate-block bitmap, so the result is already a
                # compressed boolean mask identical to a full scan's.  Return it
                # as-is (no positions round-trip) when no null post-filtering is
                # needed; the caller views it directly and extracts positions
                # lazily.  Only columns whose null sentinel can satisfy the
                # predicate (non-NaN) require the positions fall-back; NaN
                # sentinels are already excluded by the predicate itself.
                if skip_null_filter or not nullable_needs_exclude:
                    try:
                        mask = expr_result.compute(_candidate_blocks=bitmap)
                    except Exception:
                        mask = None
                    if isinstance(mask, blosc2.NDArray) and mask.dtype == np.bool_:
                        return mask
                positions = self._block_pruned_positions(expr_result, bitmap, primary_col_arr)
                if positions is not None:
                    return positions if skip_null_filter else _exclude_null_positions(positions)
            _, positions = evaluate_segment_query(
                expression, merged_operands, {}, where_dict, plan, return_positions=True
            )
            return positions if skip_null_filter else _exclude_null_positions(positions)

        return None
