Skip to content

API: Metrics & helpers

The metrics take probabilities of shape (B, C, *spatial) and integer labels of shape (B, *spatial), with optional mask and ignore_index.

negative_log_likelihood 

negative_log_likelihood(probs: Any, targets: Any, mask: Any | None = None, ignore_index: int = -100) -> float

Mean negative log-likelihood (cross-entropy) over valid voxels.

Source code in src/fiducio/metrics/calibration.py 
36
37
38
39
40
41
42
43
44
45
46
47
def negative_log_likelihood(
    probs: Any,
    targets: Any,
    mask: Any | None = None,
    ignore_index: int = -100,
) -> float:
    """Mean negative log-likelihood (cross-entropy) over valid voxels."""
    p, y = _flatten_valid(probs, targets, mask, ignore_index)
    if p.shape[0] == 0:
        return float("nan")
    true_p = p.gather(1, y.unsqueeze(1)).squeeze(1)
    return float((-safe_log(true_p)).mean().item())

expected_calibration_error 

expected_calibration_error(probs: Any, targets: Any, mask: Any | None = None, ignore_index: int = -100, n_bins: int = 15) -> float

Top-1 expected calibration error (ECE) with uniform binning.

The confidence is the maximum predicted probability and the accuracy is whether the argmax matches the label. Bins partition [0, 1] uniformly and each bin's gap is weighted by its share of voxels — see :func:average_calibration_error for the unweighted variant.

n_bins defaults to 15. The paper reports ECE with n_bins=50 (ACE with n_bins=15) and averages metrics per image before pooling; pass n_bins=50 and compute per case to follow that convention. Boundary-aware ECE is not included.

Source code in src/fiducio/metrics/calibration.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def expected_calibration_error(
    probs: Any,
    targets: Any,
    mask: Any | None = None,
    ignore_index: int = -100,
    n_bins: int = 15,
) -> float:
    """Top-1 expected calibration error (ECE) with uniform binning.

    The confidence is the maximum predicted probability and the accuracy is
    whether the argmax matches the label. Bins partition ``[0, 1]`` uniformly
    and each bin's gap is weighted by its share of voxels — see
    :func:`average_calibration_error` for the unweighted variant.

    ``n_bins`` defaults to 15. The paper reports ECE with ``n_bins=50`` (ACE with
    ``n_bins=15``) and averages metrics per image before pooling; pass
    ``n_bins=50`` and compute per case to follow that convention. Boundary-aware
    ECE is not included.
    """
    return reliability_curve(probs, targets, mask, ignore_index, n_bins).ece

average_calibration_error 

average_calibration_error(probs: Any, targets: Any, mask: Any | None = None, ignore_index: int = -100, n_bins: int = 15) -> float

Top-1 average calibration error (ACE) with uniform binning.

Uses the same uniform confidence bins as :func:expected_calibration_error, but averages the per-bin |confidence - accuracy| gap unweighted over non-empty bins instead of weighting each bin by its share of voxels. A confidence region visited by only a handful of voxels therefore counts as much as a densely populated one, which ECE would otherwise drown out. n_bins=15 matches the paper's ACE; the paper computes ACE on the pooled test voxels rather than per image.

Source code in src/fiducio/metrics/calibration.py 
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
def average_calibration_error(
    probs: Any,
    targets: Any,
    mask: Any | None = None,
    ignore_index: int = -100,
    n_bins: int = 15,
) -> float:
    """Top-1 average calibration error (ACE) with uniform binning.

    Uses the same uniform confidence bins as :func:`expected_calibration_error`,
    but averages the per-bin ``|confidence - accuracy|`` gap **unweighted**
    over non-empty bins instead of weighting each bin by its share of voxels.
    A confidence region visited by only a handful of voxels therefore counts as
    much as a densely populated one, which ECE would otherwise drown out.
    ``n_bins=15`` matches the paper's ACE; the paper computes ACE on the pooled
    test voxels rather than per image.
    """
    return reliability_curve(probs, targets, mask, ignore_index, n_bins).ace

brier_score 

brier_score(probs: Any, targets: Any, mask: Any | None = None, ignore_index: int = -100) -> float

Mean multiclass Brier score over valid voxels.

Source code in src/fiducio/metrics/calibration.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
def brier_score(
    probs: Any,
    targets: Any,
    mask: Any | None = None,
    ignore_index: int = -100,
) -> float:
    """Mean multiclass Brier score over valid voxels."""
    p, y = _flatten_valid(probs, targets, mask, ignore_index)
    if p.shape[0] == 0:
        return float("nan")
    one_hot = torch.zeros_like(p)
    one_hot.scatter_(1, y.unsqueeze(1), 1.0)
    return float((p - one_hot).square().sum(dim=1).mean().item())

reliability_curve 

reliability_curve(probs: Any, targets: Any, mask: Any | None = None, ignore_index: int = -100, n_bins: int = 15) -> ReliabilityCurve

Compute top-1 reliability statistics with uniform binning.

The confidence is the maximum predicted probability and the accuracy is whether the argmax matches the label. Useful both for reporting ECE/ACE and for drawing reliability diagrams (see :func:fiducio.plots.reliability_diagram).

Source code in src/fiducio/metrics/calibration.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def reliability_curve(
    probs: Any,
    targets: Any,
    mask: Any | None = None,
    ignore_index: int = -100,
    n_bins: int = 15,
) -> ReliabilityCurve:
    """Compute top-1 reliability statistics with uniform binning.

    The confidence is the maximum predicted probability and the accuracy is
    whether the argmax matches the label. Useful both for reporting ECE/ACE and
    for drawing reliability diagrams (see
    :func:`fiducio.plots.reliability_diagram`).
    """
    if isinstance(n_bins, bool) or not isinstance(n_bins, int) or n_bins < 1:
        raise ValueError("n_bins must be an integer >= 1")
    p, y = _flatten_valid(probs, targets, mask, ignore_index)
    # Construct the same float32 boundaries as before, then accumulate in double.
    edges = torch.linspace(0.0, 1.0, n_bins + 1, device=p.device).double()
    if p.shape[0] == 0:
        zeros = torch.zeros(n_bins, dtype=torch.float64, device=p.device)
        return ReliabilityCurve(
            edges, zeros, zeros.clone(), zeros.long(), float("nan"), float("nan")
        )

    confidence, prediction = p.max(dim=1)
    confidence = confidence.double()
    correct = (prediction == y).double()
    # Bin index in [0, n_bins - 1].
    idx = torch.bucketize(confidence, edges[1:-1].contiguous(), right=False)

    counts = torch.zeros(n_bins, dtype=torch.int64, device=p.device).scatter_add_(
        0, idx, torch.ones_like(idx)
    )
    sum_conf = torch.zeros(n_bins, dtype=torch.float64, device=p.device).scatter_add_(0, idx, confidence)
    sum_acc = torch.zeros_like(sum_conf).scatter_add_(0, idx, correct)
    safe_counts = counts.clamp_min(1.0)
    bin_conf = sum_conf / safe_counts
    bin_acc = sum_acc / safe_counts
    total = float(confidence.shape[0])
    gap = (bin_conf - bin_acc).abs()
    ece = float(((counts.double() / total) * gap).sum().item())
    nonempty = counts > 0
    ace = float(gap[nonempty].mean().item()) if bool(nonempty.any()) else float("nan")
    return ReliabilityCurve(edges, bin_conf, bin_acc, counts, ece, ace)

ReliabilityCurve dataclass 

ReliabilityCurve(bin_edges: Tensor, bin_confidence: Tensor, bin_accuracy: Tensor, bin_counts: Tensor, ece: float, ace: float)

Per-bin reliability statistics from top-1 confidence binning.

Attributes:

Name Type Description
bin_edges Tensor

(n_bins + 1,) bin boundaries in [0, 1].
bin_confidence Tensor

(n_bins,) mean predicted confidence per bin (0 for empty bins).
bin_accuracy Tensor

(n_bins,) mean accuracy per bin (0 for empty bins).
bin_counts Tensor

(n_bins,) int64 number of voxels per bin. Other statistics are float64; all tensors reside on the predictions' device.
ece float

Expected calibration error (count-weighted mean |confidence - accuracy| over bins).
ace float

Average calibration error (unweighted mean |confidence - accuracy| over non-empty bins). Unlike ece, a sparsely populated bin counts as much as a densely populated one.

Helpers

two_channel_from_binary 

two_channel_from_binary(scores: ArrayLike, *, input_type: str = 'logits') -> torch.Tensor

Convert single-channel binary outputs to the two-channel form Fiducio uses.

Fiducio represents binary segmentation as two channels (C = 2). Use this helper to convert a single-channel sigmoid output, whose class axis at dimension 1 has size 1 (shape (B, 1, *spatial)), into (B, 2, *spatial). Class 1 is the positive class.

Parameters:

Name Type Description Default
scores ArrayLike

(B, 1, *spatial) score for the positive class.

 required
input_type str

"logits" — a sigmoid logit z; returns logits [0, z] whose softmax equals [1 - sigmoid(z), sigmoid(z)]. "probs" — a probability p; returns [1 - p, p].

 'logits'
Source code in src/fiducio/utils/tensors.py 
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
def two_channel_from_binary(
    scores: ArrayLike, *, input_type: str = "logits"
) -> torch.Tensor:
    """Convert single-channel binary outputs to the two-channel form Fiducio uses.

    Fiducio represents binary segmentation as two channels (``C = 2``). Use this
    helper to convert a single-channel sigmoid output, whose class axis at
    dimension 1 has size 1 (shape ``(B, 1, *spatial)``), into ``(B, 2, *spatial)``.
    Class 1 is the positive class.

    Parameters
    ----------
    scores:
        ``(B, 1, *spatial)`` score for the positive class.
    input_type:
        ``"logits"`` — a sigmoid logit ``z``; returns logits ``[0, z]`` whose
        softmax equals ``[1 - sigmoid(z), sigmoid(z)]``. ``"probs"`` — a
        probability ``p``; returns ``[1 - p, p]``.
    """
    x = to_tensor(scores, dtype=torch.float32)
    if x.ndim < 2 or x.shape[1] != 1:
        raise ValueError(
            "scores must have a singleton class axis at dimension 1, i.e. shape "
            f"(B, 1, *spatial); got {tuple(x.shape)}"
        )
    if input_type == "logits":
        return torch.cat([torch.zeros_like(x), x], dim=1)
    if input_type == "probs":
        return torch.cat([1.0 - x, x], dim=1)
    raise ValueError(f"input_type must be 'logits' or 'probs', got {input_type!r}")

get_calibrator_class 

get_calibrator_class(calibrator_id: str) -> type[Calibrator]

Return the calibrator class registered under calibrator_id.

Source code in src/fiducio/registry.py 
41
42
43
44
45
46
47
48
49
def get_calibrator_class(calibrator_id: str) -> type[Calibrator]:
    """Return the calibrator class registered under ``calibrator_id``."""
    try:
        return _REGISTRY[calibrator_id]
    except KeyError:
        raise KeyError(
            f"unknown calibrator id {calibrator_id!r}; known ids: "
            f"{sorted(_REGISTRY)}"
        ) from None

registered_ids 

registered_ids() -> list[str]

Return the sorted list of registered calibrator ids.

Source code in src/fiducio/registry.py 
52
53
54
def registered_ids() -> list[str]:
    """Return the sorted list of registered calibrator ids."""
    return sorted(_REGISTRY)