deeplabcut.core.inferenceutils

Classes:

Name	Description
MatchedPrediction	A match between a prediction and a ground truth assembly.

Functions:

Name	Description
evaluate_assembly_greedy	Runs greedy mAP evaluation, as done by pycocotools.
match_assemblies	Matches assemblies to ground truth predictions.

MatchedPrediction dataclass

A match between a prediction and a ground truth assembly.

The ground truth assembly should be None f the prediction was not matched to any GT, and the OKS should be 0.

Attributes:

Name	Type	Description
prediction	Assembly	A prediction made by a pose model.
score	float	The confidence score for the prediction.
ground_truth	Assembly | None	If None, then this prediction is not matched to any ground truth (this can happen when there are more predicted individuals than GT). Otherwise, the ground truth assembly to which this prediction is matched.
oks	float	The OKS score between the prediction and the ground truth pose.

Source code in deeplabcut/core/inferenceutils.py

@dataclass
class MatchedPrediction:
    """A match between a prediction and a ground truth assembly.

    The ground truth assembly should be None f the prediction was not matched to any GT,
    and the OKS should be 0.

    Attributes:
        prediction: A prediction made by a pose model.
        score: The confidence score for the prediction.
        ground_truth: If None, then this prediction is not matched to any ground truth
            (this can happen when there are more predicted individuals than GT).
            Otherwise, the ground truth assembly to which this prediction is matched.
        oks: The OKS score between the prediction and the ground truth pose.
    """

    prediction: Assembly
    score: float
    ground_truth: Assembly | None
    oks: float

evaluate_assembly_greedy

evaluate_assembly_greedy(
    assemblies_gt: dict[Any, list[Assembly]],
    assemblies_pred: dict[Any, list[Assembly]],
    oks_sigma: float,
    oks_thresholds: Iterable[float],
    margin: int | float = 0,
    symmetric_kpts: list[tuple[int, int]] | None = None,
) -> dict

Runs greedy mAP evaluation, as done by pycocotools.

Parameters:

Name	Type	Description	Default
assemblies_gt	dict[Any, list[Assembly]]	A dictionary mapping image ID (e.g. filepath) to ground truth assemblies. Should contain all the same keys as assemblies_pred.	required
assemblies_pred	dict[Any, list[Assembly]]	A dictionary mapping image ID (e.g. filepath) to predicted assemblies. Should contain all the same keys as assemblies_gt.	required
oks_sigma	float	The sigma to use to compute OKS values for keypoints .	required
oks_thresholds	Iterable[float]	The OKS thresholds at which to compute precision & recall.	required
margin	int | float	The margin to use to compute bounding boxes from keypoints.	0
symmetric_kpts	list[tuple[int, int]] | None	The symmetric keypoints in the dataset.	None

Source code in deeplabcut/core/inferenceutils.py

def evaluate_assembly_greedy(
    assemblies_gt: dict[Any, list[Assembly]],
    assemblies_pred: dict[Any, list[Assembly]],
    oks_sigma: float,
    oks_thresholds: Iterable[float],
    margin: int | float = 0,
    symmetric_kpts: list[tuple[int, int]] | None = None,
) -> dict:
    """Runs greedy mAP evaluation, as done by pycocotools.

    Args:
        assemblies_gt: A dictionary mapping image ID (e.g. filepath) to ground truth
            assemblies. Should contain all the same keys as ``assemblies_pred``.
        assemblies_pred: A dictionary mapping image ID (e.g. filepath) to predicted
            assemblies. Should contain all the same keys as ``assemblies_gt``.
        oks_sigma: The sigma to use to compute OKS values for keypoints .
        oks_thresholds: The OKS thresholds at which to compute precision & recall.
        margin: The margin to use to compute bounding boxes from keypoints.
        symmetric_kpts: The symmetric keypoints in the dataset.
    """
    recall_thresholds = np.linspace(  # np.linspace(0, 1, 101)
        start=0.0, stop=1.00, num=int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True
    )
    precisions = []
    recalls = []
    for oks_t in oks_thresholds:
        all_matched = []
        total_gt_assemblies = 0
        for ind, gt_assembly in assemblies_gt.items():
            pred_assemblies = assemblies_pred.get(ind, [])
            num_gt_assemblies, matched = match_assemblies(
                pred_assemblies,
                gt_assembly,
                oks_sigma,
                margin,
                symmetric_kpts,
                greedy_matching=True,
                greedy_oks_threshold=oks_t,
            )
            all_matched.extend(matched)
            total_gt_assemblies += num_gt_assemblies

        if len(all_matched) == 0:
            precisions.append(0.0)
            recalls.append(0.0)
            continue

        # Global sort of assemblies (across all images) by score
        scores = np.asarray([-m.score for m in all_matched])
        sorted_pred_indices = np.argsort(scores, kind="mergesort")
        oks = np.asarray([match.oks for match in all_matched])[sorted_pred_indices]

        # Compute prediction and recall
        p, r = _compute_precision_and_recall(total_gt_assemblies, oks, oks_t, recall_thresholds)
        precisions.append(p)
        recalls.append(r)

    precisions = np.asarray(precisions)
    recalls = np.asarray(recalls)
    return {
        "precisions": precisions,
        "recalls": recalls,
        "mAP": precisions.mean(),
        "mAR": recalls.mean(),
    }

match_assemblies

match_assemblies(
    predictions: list[Assembly],
    ground_truth: list[Assembly],
    sigma: float,
    margin: int = 0,
    symmetric_kpts: list[tuple[int, int]] | None = None,
    greedy_matching: bool = False,
    greedy_oks_threshold: float = 0.0,
) -> tuple[int, list[MatchedPrediction]]

Matches assemblies to ground truth predictions.

Returns:

Name	Type	Description
int	tuple[int, list[MatchedPrediction]]	the total number of valid ground truth assemblies list[MatchedPrediction]: a list containing all valid predictions, potentially matched to ground truth assemblies.

Source code in deeplabcut/core/inferenceutils.py

def match_assemblies(
    predictions: list[Assembly],
    ground_truth: list[Assembly],
    sigma: float,
    margin: int = 0,
    symmetric_kpts: list[tuple[int, int]] | None = None,
    greedy_matching: bool = False,
    greedy_oks_threshold: float = 0.0,
) -> tuple[int, list[MatchedPrediction]]:
    """Matches assemblies to ground truth predictions.

    Returns:
        int: the total number of valid ground truth assemblies
        list[MatchedPrediction]: a list containing all valid predictions, potentially
            matched to ground truth assemblies.
    """
    # Only consider assemblies of at least two keypoints
    predictions = [a for a in predictions if len(a) > 1]
    ground_truth = [a for a in ground_truth if len(a) > 1]
    num_ground_truth = len(ground_truth)

    # Sort predictions by score
    inds_pred = np.argsort([ins.affinity if ins.n_links else ins.confidence for ins in predictions])[::-1]
    predictions = np.asarray(predictions)[inds_pred]

    # indices of unmatched ground truth assemblies
    matched = [
        MatchedPrediction(
            prediction=p,
            score=(p.affinity if p.n_links else p.confidence),
            ground_truth=None,
            oks=0.0,
        )
        for p in predictions
    ]

    # Greedy assembly matching like in pycocotools
    if greedy_matching:
        matched_gt_indices = set()
        for idx, pred in enumerate(predictions):
            oks = [
                calc_object_keypoint_similarity(
                    pred.xy,
                    gt.xy,
                    sigma,
                    margin,
                    symmetric_kpts,
                )
                for gt in ground_truth
            ]
            if np.all(np.isnan(oks)):
                continue

            ind_best = np.nanargmax(oks)

            # if this gt already matched, and not a crowd, continue
            if ind_best in matched_gt_indices:
                continue

            # Only match the pred to the GT if the OKS value is above a given threshold
            if oks[ind_best] < greedy_oks_threshold:
                continue

            matched_gt_indices.add(ind_best)
            matched[idx].ground_truth = ground_truth[ind_best]
            matched[idx].oks = oks[ind_best]

    # Global rather than greedy assembly matching
    else:
        inds_true = list(range(len(ground_truth)))
        mat = np.zeros((len(predictions), len(ground_truth)))
        for i, a_pred in enumerate(predictions):
            for j, a_true in enumerate(ground_truth):
                oks = calc_object_keypoint_similarity(
                    a_pred.xy,
                    a_true.xy,
                    sigma,
                    margin,
                    symmetric_kpts,
                )
                if ~np.isnan(oks):
                    mat[i, j] = oks
        rows, cols = linear_sum_assignment(mat, maximize=True)
        for row, col in zip(rows, cols, strict=False):
            matched[row].ground_truth = ground_truth[col]
            matched[row].oks = mat[row, col]
            _ = inds_true.remove(col)

    return num_ground_truth, matched