deeplabcut.refine_training_dataset.outlier_frames

Functions:

Name	Description
PlottingSingleFrame	Label frame and save under imagename / this is already cropped (for clip)
PlottingSingleFramecv2	Label frame and save under imagename / cap is not already cropped.
attempt_to_add_video	Add new videos to the config file at any stage of the project.
compute_deviations	Fits Seasonal AutoRegressive Integrated Moving Average with eXogenous regressors
convertparms2start	Creating a start value for sarimax in case of an value error
extract_outlier_frames	Extracts the outlier frames.
find_outliers_in_raw_data	Extract outlier frames from either raw detections or assemblies of multiple
find_outliers_in_raw_detections	Find outlier frames from the raw detections of multiple animals.
merge_datasets	Merge the original training dataset with the newly refined data.

PlottingSingleFrame

PlottingSingleFrame(
    clip,
    Dataframe,
    bodyparts2plot,
    tmpfolder,
    index,
    dotsize,
    pcutoff,
    alphavalue,
    colors,
    strwidth=4,
    savelabeled=True,
)

Label frame and save under imagename / this is already cropped (for clip)

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def PlottingSingleFrame(
    clip,
    Dataframe,
    bodyparts2plot,
    tmpfolder,
    index,
    dotsize,
    pcutoff,
    alphavalue,
    colors,
    strwidth=4,
    savelabeled=True,
):
    """Label frame and save under imagename / this is already cropped (for clip)"""
    from skimage import io

    imagename1 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + ".png")
    imagename2 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + "labeled.png")

    if not imagename1.is_file():
        plt.axis("off")
        image = img_as_ubyte(clip.get_frame(index * 1.0 / clip.fps))
        io.imsave(str(imagename1), image)

        if savelabeled:
            if np.ndim(image) > 2:
                h, w, nc = np.shape(image)
            else:
                h, w = np.shape(image)

            bpts = Dataframe.columns.get_level_values("bodyparts")
            all_bpts = bpts.values[::3]
            df_x, df_y, df_likelihood = Dataframe.values.reshape((Dataframe.shape[0], -1, 3)).T
            bplist = bpts.unique().to_list()
            if Dataframe.columns.nlevels == 3:
                map2bp = list(range(len(all_bpts)))
            else:
                map2bp = [bplist.index(bp) for bp in all_bpts]
            keep = np.flatnonzero(np.isin(all_bpts, bodyparts2plot))

            plt.figure(frameon=False, figsize=(w * 1.0 / 100, h * 1.0 / 100))
            plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
            plt.imshow(image)
            for i, ind in enumerate(keep):
                if df_likelihood[ind, index] > pcutoff:
                    plt.scatter(
                        df_x[ind, index],
                        df_y[ind, index],
                        s=dotsize**2,
                        color=colors(map2bp[i]),
                        alpha=alphavalue,
                    )
            plt.xlim(0, w)
            plt.ylim(0, h)
            plt.axis("off")
            plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
            plt.gca().invert_yaxis()
            plt.savefig(imagename2)
            plt.close("all")

PlottingSingleFramecv2

PlottingSingleFramecv2(
    cap, Dataframe, bodyparts2plot, tmpfolder, index, dotsize, pcutoff, alphavalue, colors, strwidth=4, savelabeled=True
)

Label frame and save under imagename / cap is not already cropped.

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def PlottingSingleFramecv2(
    cap,
    Dataframe,
    bodyparts2plot,
    tmpfolder,
    index,
    dotsize,
    pcutoff,
    alphavalue,
    colors,
    strwidth=4,
    savelabeled=True,
):
    """Label frame and save under imagename / cap is not already cropped."""
    from skimage import io

    imagename1 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + ".png")
    imagename2 = Path(tmpfolder) / ("img" + str(index).zfill(strwidth) + "labeled.png")

    if not imagename1.is_file():
        plt.axis("off")
        cap.set_to_frame(index)
        frame = cap.read_frame(crop=True)
        if frame is None:
            print("Frame could not be read.")
            return
        image = img_as_ubyte(frame)
        io.imsave(imagename1, image)

        if savelabeled:
            if np.ndim(image) > 2:
                h, w, nc = np.shape(image)
            else:
                h, w = np.shape(image)

            bpts = Dataframe.columns.get_level_values("bodyparts")
            all_bpts = bpts.values[::3]
            df_x, df_y, df_likelihood = Dataframe.values.reshape((Dataframe.shape[0], -1, 3)).T
            bplist = bpts.unique().to_list()
            if Dataframe.columns.nlevels == 3:
                map2bp = list(range(len(all_bpts)))
            else:
                map2bp = [bplist.index(bp) for bp in all_bpts]
            keep = np.flatnonzero(np.isin(all_bpts, bodyparts2plot))

            plt.figure(frameon=False, figsize=(w * 1.0 / 100, h * 1.0 / 100))
            plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
            plt.imshow(image)
            for i, ind in enumerate(keep):
                if df_likelihood[ind, index] > pcutoff:
                    plt.scatter(
                        df_x[ind, index],
                        df_y[ind, index],
                        s=dotsize**2,
                        color=colors(map2bp[i]),
                        alpha=alphavalue,
                    )
            plt.xlim(0, w)
            plt.ylim(0, h)
            plt.axis("off")
            plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
            plt.gca().invert_yaxis()
            plt.savefig(imagename2)
            plt.close("all")

attempt_to_add_video

attempt_to_add_video(config: str, video: str, copy_videos: bool, coords: list | None) -> bool

Add new videos to the config file at any stage of the project.

Parameters:

Name	Type	Description	Default
config	string	Full path of the config file in the project.	required
video	string	Full path of the video to add to the project.	required
copy_videos	bool	If this is set to True, the videos will be copied to the project/videos directory. If False, the symlink of the videos will be copied instead. The default is False; if provided it must be either True or False.	required
coords	list	A list containing the list of cropping coordinates of the video. Defaults to None.	required

Returns:

Name	Type	Description
bool	bool	True iff the video was successfully added to the project.

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def attempt_to_add_video(
    config: str,
    video: str,
    copy_videos: bool,
    coords: list | None,
) -> bool:
    """Add new videos to the config file at any stage of the project.

    Args:
        config (string): Full path of the config file in the project.
        video (string): Full path of the video to add to the project.
        copy_videos (bool, optional): If this is set to True, the videos will be copied
            to the project/videos directory. If False, the symlink of the videos will
            be copied instead. The default is ``False``; if provided it must be either
            ``True`` or ``False``.
        coords (list, optional): A list containing the list of cropping coordinates of
            the video. Defaults to None.

    Returns:
        bool: True iff the video was successfully added to the project.
    """
    from deeplabcut.create_project import add

    # make sure coords and videos are a list
    videos = [video]
    if coords is not None:
        coords = [coords]

    try:
        add.add_new_videos(config, videos, coords=coords, copy_videos=copy_videos)
    except Exception:
        # can we make a catch here? - in fact we should drop indices from DataCombined
        # if they are in CollectedData.. [ideal behavior; currently pretty unlikely]
        print(
            "AUTOMATIC ADDING OF VIDEO TO CONFIG FILE FAILED! You need to "
            "do this manually for including it in the config.yaml file!"
        )
        print("Videopath:", video, "Coordinates for cropping:", coords)
        return False

    return True

compute_deviations

compute_deviations(Dataframe, dataname, p_bound, alpha, ARdegree, MAdegree, storeoutput=None)

Fits Seasonal AutoRegressive Integrated Moving Average with eXogenous regressors model to data and computes confidence interval as well as mean fit.

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def compute_deviations(Dataframe, dataname, p_bound, alpha, ARdegree, MAdegree, storeoutput=None):
    """Fits Seasonal AutoRegressive Integrated Moving Average with eXogenous regressors
    model to data and computes confidence interval as well as mean fit."""

    print("Fitting state-space models with parameters:", ARdegree, MAdegree)
    df_x, df_y, df_likelihood = Dataframe.values.reshape((Dataframe.shape[0], -1, 3)).T
    preds = []
    for row in range(len(df_x)):
        x = df_x[row]
        y = df_y[row]
        p = df_likelihood[row]
        meanx, CIx = FitSARIMAXModel(x, p, p_bound, alpha, ARdegree, MAdegree)
        meany, CIy = FitSARIMAXModel(y, p, p_bound, alpha, ARdegree, MAdegree)
        distance = np.sqrt((x - meanx) ** 2 + (y - meany) ** 2)
        significant = (x < CIx[:, 0]) + (x > CIx[:, 1]) + (y < CIy[:, 0]) + (y > CIy[:, 1])
        preds.append(np.c_[distance, significant, meanx, meany, CIx, CIy])

    columns = Dataframe.columns
    # Use the existing valid keypoint combinations, in their original order.
    # The goal is to extract each stream (e.g. Scorer/ID/Bodypart) as a separate column,
    # and then build, for each stat, a MultiIndex with the same levels, i.e.
    # Scorer/ID/Bodypart/stat (see stats below).
    # Note, this could be built from "y" as well without any difference in the output
    base_cols = Dataframe.xs("x", axis=1, level="coords", drop_level=True).columns
    stats = [
        "distance",
        "sig",
        "meanx",
        "meany",
        "lowerCIx",
        "higherCIx",
        "lowerCIy",
        "higherCIy",
    ]
    pdindex = pd.MultiIndex.from_tuples(
        [(*col, stat) for col in base_cols for stat in stats],
        names=[n for n in columns.names if n != "coords"] + ["stats"],
    )
    data = pd.DataFrame(np.concatenate(preds, axis=1), columns=pdindex)  # preds (n_frames, n_stats * n_streams)
    # average distance and average # significant differences avg. over comparisonbodyparts
    d = data.xs("distance", axis=1, level=-1).mean(axis=1).values
    o = data.xs("sig", axis=1, level=-1).mean(axis=1).values

    if storeoutput == "full":
        data.to_hdf(
            dataname.split(".h5")[0] + "filtered.h5",
            key="df_with_missing",
            format="table",
            mode="w",
        )
        return d, o, data
    else:
        return d, o

convertparms2start

convertparms2start(pn)

Creating a start value for sarimax in case of an value error See: https://groups.google.com/forum/#!topic/pystatsmodels/S_Fo53F25Rk

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def convertparms2start(pn):
    """Creating a start value for sarimax in case of an value error
    See: https://groups.google.com/forum/#!topic/pystatsmodels/S_Fo53F25Rk"""
    if "ar." in pn:
        return 0
    elif "ma." in pn:
        return 0
    elif "sigma" in pn:
        return 1
    else:
        return 0

extract_outlier_frames

extract_outlier_frames(
    config: str | Path,
    videos: list[str | Path],
    video_extensions: str | Sequence[str] | None = None,
    shuffle=1,
    trainingsetindex=0,
    outlieralgorithm="jump",
    frames2use=None,
    comparisonbodyparts="all",
    epsilon=20,
    p_bound=0.01,
    ARdegree=3,
    MAdegree=1,
    alpha=0.01,
    extractionalgorithm="kmeans",
    automatic=False,
    cluster_resizewidth=30,
    cluster_color=False,
    opencv=True,
    savelabeled=False,
    copy_videos=False,
    destfolder=None,
    modelprefix="",
    track_method="",
    **kwargs
)

Extracts the outlier frames.

Extracts the outlier frames if the predictions are not correct for a certain video from the cropped video running from start to stop as defined in config.yaml.

Another crucial parameter in config.yaml is how many frames to extract numframes2extract.

Parameters:

Name	Type	Description	Default
config	str | Path	Full path of the config.yaml file.	required
videos	list[str | Path]	The full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.	required
video_extensions	str | Sequence[str] | None	Controls how videos are filtered, based on file extension. File paths and directory contents are treated differently: - None (default): file paths are accepted as-is; directories are scanned for files with a recognized video extension. - str or Sequence[str] (e.g. "mp4" or ["mp4", "avi"]): both file paths and directory contents are filtered by the given extension(s). Defaults to None.	None
shuffle	int	The shuffle index of training dataset. The extracted frames will be stored in the labeled-dataset for the corresponding shuffle of training dataset. Defaults to 1.	1
trainingsetindex	int	Integer specifying which TrainingsetFraction to use. Note that TrainingFraction is a list in config.yaml. Defaults to 0.	0
outlieralgorithm	str	String specifying the algorithm used to detect the outliers. 'fitting' fits an Auto Regressive Integrated Moving Average model to the data and computes the distance to the estimated data. Larger distances than epsilon are then potentially identified as outliers 'jump' identifies larger jumps than 'epsilon' in any body part 'uncertain' looks for frames with confidence below p_bound 'manual' launches a GUI from which the user can choose the frames 'list' looks for user to provide a list of frame numbers to use, 'frames2use'. In this case, 'extractionalgorithm' is forced to be 'uniform.' Defaults to "jump".	'jump'
frames2use	list[str]	If 'outlieralgorithm' is 'list', provide the list of frames here. Defaults to None.	None
comparisonbodyparts	list[str] or str	This selects the body parts for which the comparisons with the outliers are carried out. If "all", then all body parts from config.yaml are used. If a list of strings that are a subset of the full list E.g. ['hand','Joystick'] for the demo Reaching-Mackenzie-2018-08-30/config.yaml to select only these body parts. Defaults to "all".	'all'
p_bound	float	For outlieralgorithm 'uncertain' this parameter defines the likelihood below which a body part will be flagged as a putative outlier. Defaults to 0.01.	0.01
epsilon	float	If 'outlieralgorithm' is 'fitting', this is the float bound according to which frames are picked when the (average) body part estimate deviates from model fit. If 'outlieralgorithm' is 'jump', this is the float bound specifying the distance by which body points jump from one frame to next (Euclidean distance). Defaults to 20.	20
ARdegree	int	For outlieralgorithm 'fitting': Autoregressive degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part) See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html Defaults to 3.	3
MAdegree	int	For outlieralgorithm 'fitting': Moving Average degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and seasonal part) See https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html Defaults to 1.	1
alpha	float	Significance level for detecting outliers based on confidence interval of fitted ARIMA model. Only the distance is used however. Defaults to 0.01.	0.01
extractionalgorithm	str	String specifying the algorithm to use for selecting the frames from the identified putatative outlier frames. Currently, deeplabcut supports either kmeans or uniform based selection (same logic as for extract_frames). Defaults to "kmeans".	'kmeans'
automatic	bool	If True, extract outliers without being asked for user feedback. Defaults to False.	False
cluster_resizewidth	number	If "extractionalgorithm" is "kmeans", one can change the width to which the images are downsampled (aspect ratio is fixed). Defaults to 30.	30
cluster_color	bool	If False, each downsampled image is treated as a grayscale vector (discarding color information). If True, then the color channels are considered. This increases the computational complexity. Defaults to False.	False
opencv	bool	Uses openCV for loading & extractiong (otherwise moviepy (legacy)). Defaults to True.	True
savelabeled	bool	If True, frame are saved with predicted labels in each folder. Defaults to False.	False
copy_videos	bool	If True, newly-added videos (from which outlier frames are extracted) are copied to the project folder. By default, symbolic links are created instead. Defaults to False.	False
destfolder	str or None	Specifies the destination folder that was used for storing analysis data. If None, the path of the video is used. Defaults to None.	None
modelprefix	str	Directory containing the deeplabcut models to use when evaluating the network. By default, the models are assumed to exist in the project folder. Defaults to "".	''
track_method	str	Specifies the tracker used to generate the data. Empty by default (corresponding to a single animal project). For multiple animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken from the config.yaml file if none is given. Defaults to "".	''
**kwargs		Additional arguments. For torch-based shuffles, can be used to specify: - snapshot_index - detector_snapshot_index	{}

Returns:

Type	Description
	None

Examples:

Extract the frames with default settings on Windows.

deeplabcut.extract_outlier_frames(
    'C:\myproject\reaching-task\config.yaml',
    ['C:\yourusername\rig-95\Videos\reachingvideo1.avi'],
)

Extract the frames with default settings on Linux/MacOS.

deeplabcut.extract_outlier_frames(
    '/analysis/project/reaching-task/config.yaml',
    ['/analysis/project/video/reachinvideo1.avi'],
)

Extract the frames using the "kmeans" algorithm.

deeplabcut.extract_outlier_frames(
    '/analysis/project/reaching-task/config.yaml',
    ['/analysis/project/video/reachinvideo1.avi'],
    extractionalgorithm='kmeans',
)

Extract the frames using the "kmeans" algorithm and "epsilon=5" pixels.

deeplabcut.extract_outlier_frames(
    '/analysis/project/reaching-task/config.yaml',
    ['/analysis/project/video/reachinvideo1.avi'],
    epsilon=5,
    extractionalgorithm='kmeans',
)

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

@renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
def extract_outlier_frames(
    config: str | Path,
    videos: list[str | Path],
    video_extensions: str | Sequence[str] | None = None,
    shuffle=1,
    trainingsetindex=0,
    outlieralgorithm="jump",
    frames2use=None,
    comparisonbodyparts="all",
    epsilon=20,
    p_bound=0.01,
    ARdegree=3,
    MAdegree=1,
    alpha=0.01,
    extractionalgorithm="kmeans",
    automatic=False,
    cluster_resizewidth=30,
    cluster_color=False,
    opencv=True,
    savelabeled=False,
    copy_videos=False,
    destfolder=None,
    modelprefix="",
    track_method="",
    **kwargs,
):
    """Extracts the outlier frames.

    Extracts the outlier frames if the predictions are not correct for a certain video
    from the cropped video running from start to stop as defined in config.yaml.

    Another crucial parameter in config.yaml is how many frames to extract
    ``numframes2extract``.

    Args:
        config (str | Path): Full path of the config.yaml file.
        videos (list[str | Path]): The full paths to videos for analysis or a path to the
            directory, where all the videos with same extension are stored.
        video_extensions (str | Sequence[str] | None, optional): Controls how ``videos`` are
            filtered, based on file extension. File paths and directory contents are
            treated differently:
            - ``None`` (default): file paths are accepted as-is; directories are
              scanned for files with a recognized video extension.
            - ``str`` or ``Sequence[str]`` (e.g. ``"mp4"`` or ``["mp4", "avi"]``):
              both file paths and directory contents are filtered by the given
              extension(s). Defaults to None.
        shuffle (int, optional): The shuffle index of training dataset. The extracted
            frames will be stored in the labeled-dataset for the corresponding shuffle
            of training dataset. Defaults to 1.
        trainingsetindex (int, optional): Integer specifying which TrainingsetFraction
            to use. Note that TrainingFraction is a list in config.yaml. Defaults to 0.
        outlieralgorithm (str, optional): String specifying the algorithm used to detect
            the outliers.

            * ``'fitting'`` fits an Auto Regressive Integrated Moving Average model to
              the data and computes the distance to the estimated data. Larger distances
              than epsilon are then potentially identified as outliers
            * ``'jump'`` identifies larger jumps than 'epsilon' in any body part
            * ``'uncertain'`` looks for frames with confidence below p_bound
            * ``'manual'`` launches a GUI from which the user can choose the frames
            * ``'list'`` looks for user to provide a list of frame numbers to use,
              'frames2use'. In this case, ``'extractionalgorithm'`` is forced to be
              ``'uniform.'``

            Defaults to "jump".
        frames2use (list[str], optional): If ``'outlieralgorithm'`` is ``'list'``,
            provide the list of frames here. Defaults to None.
        comparisonbodyparts (list[str] or str, optional): This selects the body parts for
            which the comparisons with the outliers are carried out. If ``"all"``, then
            all body parts from config.yaml are used. If a list of strings that are a
            subset of the full list E.g. ['hand','Joystick'] for the demo
            Reaching-Mackenzie-2018-08-30/config.yaml to select only these body parts.
            Defaults to "all".
        p_bound (float, optional): For outlieralgorithm ``'uncertain'`` this parameter
            defines the likelihood below which a body part will be flagged as a putative
            outlier. Defaults to 0.01.
        epsilon (float, optional): If ``'outlieralgorithm'`` is ``'fitting'``, this is
            the float bound according to which frames are picked when the (average) body
            part estimate deviates from model fit. If ``'outlieralgorithm'`` is
            ``'jump'``, this is the float bound specifying the distance by which body
            points jump from one frame to next (Euclidean distance). Defaults to 20.
        ARdegree (int, optional): For outlieralgorithm ``'fitting'``: Autoregressive
            degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and
            seasonal part) See
            https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
            Defaults to 3.
        MAdegree (int, optional): For outlieralgorithm ``'fitting'``: Moving Average
            degree of ARIMA model degree. (Note we use SARIMAX without exogeneous and
            seasonal part) See
            https://www.statsmodels.org/dev/generated/statsmodels.tsa.statespace.sarimax.SARIMAX.html
            Defaults to 1.
        alpha (float, optional): Significance level for detecting outliers based on
            confidence interval of fitted ARIMA model. Only the distance is used
            however. Defaults to 0.01.
        extractionalgorithm (str, optional): String specifying the algorithm to use for
            selecting the frames from the identified putatative outlier frames.
            Currently, deeplabcut supports either ``kmeans`` or ``uniform`` based
            selection (same logic as for extract_frames). Defaults to "kmeans".
        automatic (bool, optional): If ``True``, extract outliers without being asked
            for user feedback. Defaults to False.
        cluster_resizewidth (number, optional): If ``"extractionalgorithm"`` is
            ``"kmeans"``, one can change the width to which the images are downsampled
            (aspect ratio is fixed). Defaults to 30.
        cluster_color (bool, optional): If ``False``, each downsampled image is treated
            as a grayscale vector (discarding color information). If ``True``, then the
            color channels are considered. This increases the computational complexity.
            Defaults to False.
        opencv (bool, optional): Uses openCV for loading & extractiong (otherwise moviepy
            (legacy)). Defaults to True.
        savelabeled (bool, optional): If ``True``, frame are saved with predicted labels
            in each folder. Defaults to False.
        copy_videos (bool, optional): If True, newly-added videos (from which outlier
            frames are extracted) are copied to the project folder. By default, symbolic
            links are created instead. Defaults to False.
        destfolder (str or None, optional): Specifies the destination folder that was
            used for storing analysis data. If ``None``, the path of the video is used.
            Defaults to None.
        modelprefix (str, optional): Directory containing the deeplabcut models to use
            when evaluating the network. By default, the models are assumed to exist in
            the project folder. Defaults to "".
        track_method (str, optional): Specifies the tracker used to generate the data.
            Empty by default (corresponding to a single animal project). For multiple
            animals, must be either 'box', 'skeleton', or 'ellipse' and will be taken
            from the config.yaml file if none is given. Defaults to "".
        **kwargs: Additional arguments. For torch-based shuffles, can be used to specify:
            - snapshot_index
            - detector_snapshot_index

    Returns:
        None

    Examples:
        Extract the frames with default settings on Windows.

            deeplabcut.extract_outlier_frames(
                'C:\\myproject\\reaching-task\\config.yaml',
                ['C:\\yourusername\\rig-95\\Videos\\reachingvideo1.avi'],
            )

        Extract the frames with default settings on Linux/MacOS.

            deeplabcut.extract_outlier_frames(
                '/analysis/project/reaching-task/config.yaml',
                ['/analysis/project/video/reachinvideo1.avi'],
            )

        Extract the frames using the "kmeans" algorithm.

            deeplabcut.extract_outlier_frames(
                '/analysis/project/reaching-task/config.yaml',
                ['/analysis/project/video/reachinvideo1.avi'],
                extractionalgorithm='kmeans',
            )

        Extract the frames using the "kmeans" algorithm and ``"epsilon=5"`` pixels.

            deeplabcut.extract_outlier_frames(
                '/analysis/project/reaching-task/config.yaml',
                ['/analysis/project/video/reachinvideo1.avi'],
                epsilon=5,
                extractionalgorithm='kmeans',
            )
    """

    cfg = auxiliaryfunctions.read_config(config)
    bodyparts = auxiliaryfunctions.intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts)
    if not len(bodyparts):
        raise ValueError("No valid bodyparts were selected.")

    track_method = auxfun_multianimal.get_track_method(cfg, track_method=track_method)

    DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name(
        cfg,
        shuffle,
        trainFraction=cfg["TrainingFraction"][trainingsetindex],
        modelprefix=modelprefix,
        **kwargs,
    )

    Videos = collect_video_paths(videos, extensions=video_extensions)
    if len(Videos) == 0:
        print("No suitable videos found in", videos)

    for video in Videos:
        if destfolder is None:
            videofolder = str(Path(video).parents[0])
        else:
            videofolder = destfolder
        vname = Path(video).stem

        try:
            df, dataname, _, _ = auxiliaryfunctions.load_analyzed_data(
                videofolder, vname, DLCscorer, track_method=track_method
            )
            metadata = auxiliaryfunctions.load_video_metadata(videofolder, vname, DLCscorer)
            nframes = len(df)
            startindex = max([int(np.floor(nframes * cfg["start"])), 0])
            stopindex = min([int(np.ceil(nframes * cfg["stop"])), nframes])
            Index = np.arange(stopindex - startindex) + startindex

            # offset if the data was cropped
            # note: When output video is also cropped, the keypoints should be shifted back.
            out_x1, out_y1 = _read_video_specific_cropping_margins(config, video)
            # TODO @deruyter92: This pattern should be refactored throughout the codebase
            # it is reading a config value that is supposed to be missing / None.
            if (metadata.get("data") or {}).get("cropping"):
                x1, _, y1, _ = metadata["data"]["cropping_parameters"]
                df.iloc[:, df.columns.get_level_values(level="coords") == "x"] += x1 - out_x1
                df.iloc[:, df.columns.get_level_values(level="coords") == "y"] += y1 - out_y1

            df = df.iloc[Index]
            mask = df.columns.get_level_values("bodyparts").isin(bodyparts)
            df_temp = df.loc[:, mask]
            Indices = []
            if outlieralgorithm == "uncertain":
                p = df_temp.xs("likelihood", level="coords", axis=1)
                ind = df_temp.index[(p < p_bound).any(axis=1)].tolist()
                Indices.extend(ind)
            elif outlieralgorithm == "jump":
                temp_dt = df_temp.diff(axis=0) ** 2
                temp_dt.drop("likelihood", axis=1, level="coords", inplace=True)
                sum_ = temp_dt.T.groupby(level="bodyparts").sum().T
                ind = df_temp.index[(sum_ > epsilon**2).any(axis=1)].tolist()
                Indices.extend(ind)
            elif outlieralgorithm == "fitting":
                d, o = compute_deviations(df_temp, dataname, p_bound, alpha, ARdegree, MAdegree)
                # Some heuristics for extracting frames based on distance:
                ind = np.flatnonzero(d > epsilon)  # time points with at least average difference of epsilon
                if (
                    len(ind) < cfg["numframes2pick"] * 2 and len(d) > cfg["numframes2pick"] * 2
                ):  # if too few points qualify, extract the most distant ones.
                    ind = np.argsort(d)[::-1][: cfg["numframes2pick"] * 2]
                Indices.extend(ind)
            elif outlieralgorithm == "manual":
                from deeplabcut.gui.widgets import launch_napari

                added_video = attempt_to_add_video(
                    config=config,
                    video=video,
                    copy_videos=copy_videos,
                    coords=None,
                )
                if added_video:
                    project_video_path = Path(cfg["project_path"]) / "videos" / Path(video).name
                    _ = launch_napari([project_video_path, dataname])
                return

            elif outlieralgorithm == "list":
                if frames2use is not None:
                    try:
                        frames2use = np.array(frames2use).astype("int")
                    except ValueError:
                        print(
                            "Could not cast frames2use into np array, "
                            "please check that frames2use is a simply a list of integers!"
                        )
                        raise
                    Indices.extend(frames2use)
                else:
                    raise ValueError('Expected list of frames2use for outlieralgorithm "list"!')
            else:
                raise ValueError(f"outlieralgorithm {outlieralgorithm} not recognized!")

            # Run always except when the outlieralgorithm == manual.
            if not outlieralgorithm == "manual":
                Indices = np.sort(list(set(Indices)))  # remove repetitions.
                print(
                    "Method ",
                    outlieralgorithm,
                    " found ",
                    len(Indices),
                    " putative outlier frames.",
                )
                print(
                    "Do you want to proceed with extracting ",
                    cfg["numframes2pick"],
                    " of those?",
                )
                if outlieralgorithm == "uncertain" or outlieralgorithm == "jump":
                    print(
                        "If this list is very large, perhaps consider changing the parameters "
                        "(start, stop, p_bound, comparisonbodyparts) or use a different method."
                    )
                elif outlieralgorithm == "fitting":
                    print(
                        "If this list is very large, perhaps consider changing the parameters "
                        "(start, stop, epsilon, ARdegree, MAdegree, alpha, comparisonbodyparts) "
                        "or use a different method."
                    )

                if not automatic:
                    askuser = input("yes/no")
                else:
                    askuser = "Ja"

                if askuser == "y" or askuser == "yes" or askuser == "Ja" or askuser == "ha":  # multilanguage support :)
                    # Now extract from those Indices!
                    ExtractFramesbasedonPreselection(
                        Indices,
                        extractionalgorithm,
                        df,
                        video,
                        cfg,
                        config,
                        opencv,
                        cluster_resizewidth,
                        cluster_color,
                        savelabeled,
                        copy_videos=copy_videos,
                    )
                else:
                    print("Nothing extracted, please change the parameters and start again...")
        except FileNotFoundError as e:
            print(e)
            print(
                "It seems the video has not been analyzed yet, or the video is not found! "
                "You can only refine the labels after the a video is analyzed. "
                "Please run 'analyze_video' first. "
                "Or, please double check your video file path"
            )

find_outliers_in_raw_data

find_outliers_in_raw_data(
    config: str | Path,
    pickle_file: str | Path,
    video_file: str | Path,
    pcutoff=0.1,
    percentiles=(5, 95),
    with_annotations=True,
    extraction_algo="kmeans",
    copy_videos=False,
)

Extract outlier frames from either raw detections or assemblies of multiple animals.

Parameters:

Name	Type	Description	Default
config	str | Path	Absolute path to the project config.yaml.	required
pickle_file	str | Path	Path to a _full.pickle or _assemblies.pickle.	required
video_file	str | Path	Path to the corresponding video file for frame extraction.	required
pcutoff	float	Detection confidence threshold below which frames are flagged as containing outliers. Only considered if raw detections are passed in. Defaults to 0.1.	0.1
percentiles	tuple	Assemblies are considered outliers if their areas are beyond the 5th and 95th percentiles. Must contain a lower and upper bound. Defaults to (5, 95).	(5, 95)
with_annotations	bool	If true, extract frames and the corresponding network predictions. Otherwise, only the frames are extracted. Defaults to True.	True
extraction_algo	string	Outlier detection algorithm. Must be either uniform or kmeans. Defaults to "kmeans".	'kmeans'
copy_videos	bool	If True, newly-added videos (from which outlier frames are extracted) are copied to the project folder. By default, symbolic links are created instead. Defaults to False.	False

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def find_outliers_in_raw_data(
    config: str | Path,
    pickle_file: str | Path,
    video_file: str | Path,
    pcutoff=0.1,
    percentiles=(5, 95),
    with_annotations=True,
    extraction_algo="kmeans",
    copy_videos=False,
):
    """Extract outlier frames from either raw detections or assemblies of multiple
    animals.

    Args:
        config (str | Path): Absolute path to the project config.yaml.
        pickle_file (str | Path): Path to a *_full.pickle or *_assemblies.pickle.
        video_file (str | Path): Path to the corresponding video file for frame extraction.
        pcutoff (float, optional): Detection confidence threshold below which frames are
            flagged as containing outliers. Only considered if raw detections are
            passed in. Defaults to 0.1.
        percentiles (tuple, optional): Assemblies are considered outliers if their areas
            are beyond the 5th and 95th percentiles. Must contain a lower and upper
            bound. Defaults to (5, 95).
        with_annotations (bool, optional): If true, extract frames and the corresponding
            network predictions. Otherwise, only the frames are extracted. Defaults to
            True.
        extraction_algo (string, optional): Outlier detection algorithm. Must be either
            ``uniform`` or ``kmeans``. Defaults to "kmeans".
        copy_videos (bool, optional): If True, newly-added videos (from which outlier
            frames are extracted) are copied to the project folder. By default, symbolic
            links are created instead. Defaults to False.
    """
    if extraction_algo not in ("kmeans", "uniform"):
        raise ValueError(f"Unsupported extraction algorithm {extraction_algo}.")

    video_name = Path(video_file).stem
    pickle_name = Path(pickle_file).stem
    if not pickle_name.startswith(video_name):
        raise ValueError("Video and pickle files do not match.")

    with Path(pickle_file).open("rb") as file:
        data = pickle.load(file)
    if pickle_file.endswith("_full.pickle"):
        inds, data = find_outliers_in_raw_detections(data, threshold=pcutoff)
        with_annotations = False
    elif pickle_file.endswith("_assemblies.pickle"):
        assemblies = dict()
        for k, lst in data.items():
            if k == "single":
                continue
            ass = []
            for vals in lst:
                a = inferenceutils.Assembly(len(vals))
                a.data = vals
                ass.append(a)
            assemblies[k] = ass
        inds = inferenceutils.find_outlier_assemblies(assemblies, qs=percentiles)
    else:
        raise OSError(f"Raw data file {pickle_file} could not be parsed.")

    cfg = auxiliaryfunctions.read_config(config)
    ExtractFramesbasedonPreselection(
        inds,
        extraction_algo,
        data,
        video=video_file,
        cfg=cfg,
        config=config,
        savelabeled=False,
        with_annotations=with_annotations,
        copy_videos=copy_videos,
    )

find_outliers_in_raw_detections

find_outliers_in_raw_detections(pickled_data, algo='uncertain', threshold=0.1, kept_keypoints=None)

Find outlier frames from the raw detections of multiple animals.

Parameters:

Name	Type	Description	Default
pickled_data	dict	Data in the *_full.pickle file obtained after analyze_videos.	required
algo	string	Outlier detection algorithm. Currently, only 'uncertain' is supported for multi-animal raw detections. Defaults to "uncertain".	'uncertain'
threshold	float	Detection confidence threshold below which frames are flagged as containing outliers. Only considered if algo==uncertain. Defaults to 0.1.	0.1
kept_keypoints	list	Indices in the list of labeled body parts to be kept of the analysis. By default, all keypoints are used for outlier search. Defaults to None.	None

Returns:

Name	Type	Description
tuple		Indices of video frames containing potential outliers, and the processed data dictionary.

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def find_outliers_in_raw_detections(pickled_data, algo="uncertain", threshold=0.1, kept_keypoints=None):
    """Find outlier frames from the raw detections of multiple animals.

    Args:
        pickled_data (dict): Data in the *_full.pickle file obtained after
            `analyze_videos`.
        algo (string, optional): Outlier detection algorithm. Currently, only 'uncertain'
            is supported for multi-animal raw detections. Defaults to "uncertain".
        threshold (float, optional): Detection confidence threshold below which frames
            are flagged as containing outliers. Only considered if `algo`==`uncertain`.
            Defaults to 0.1.
        kept_keypoints (list, optional): Indices in the list of labeled body parts to
            be kept of the analysis. By default, all keypoints are used for outlier
            search. Defaults to None.

    Returns:
        tuple: Indices of video frames containing potential outliers, and the processed
            data dictionary.
    """
    if algo != "uncertain":
        raise ValueError("Only method 'uncertain' is currently supported.")

    try:
        _ = pickled_data.pop("metadata")
    except KeyError:
        pass

    def get_frame_ind(s):
        return int(re.findall(r"\d+", s)[0])

    candidates = []
    data = dict()
    for frame_name, dict_ in pickled_data.items():
        frame_ind = get_frame_ind(frame_name)
        temp_coords = dict_["coordinates"][0]
        temp = dict_["confidence"]
        if kept_keypoints is not None:
            temp_coords = [temp_coords[i] for i in kept_keypoints]
            temp = [temp[i] for i in kept_keypoints]
        coords = np.concatenate(temp_coords)
        conf = np.concatenate(temp)
        data[frame_ind] = np.c_[coords, conf].squeeze()
        if np.any(conf < threshold):
            candidates.append(frame_ind)
    return candidates, data

merge_datasets

merge_datasets(config: str | Path, forceiterate=None)

Merge the original training dataset with the newly refined data.

Checks if the original training dataset can be merged with the newly refined training dataset. To do so it will check if the frames in all extracted video sets were relabeled.

If this is the case then the "iteration" variable is advanced by 1.

Parameters:

Name	Type	Description	Default
config	str | Path	Full path of the config.yaml file.	required
forceiterate	int or None	If an integer is given the iteration variable is set to this value. This is only done if all datasets were labeled or refined. Defaults to None.	None

Examples:

    deeplabcut.merge_datasets("/analysis/project/reaching-task/config.yaml")

Source code in deeplabcut/refine_training_dataset/outlier_frames.py

def merge_datasets(config: str | Path, forceiterate=None):
    """Merge the original training dataset with the newly refined data.

    Checks if the original training dataset can be merged with the newly refined
    training dataset. To do so it will check if the frames in all extracted video sets
    were relabeled.

    If this is the case then the ``"iteration"`` variable is advanced by 1.

    Args:
        config (str | Path): Full path of the config.yaml file.
        forceiterate (int or None, optional): If an integer is given the iteration
            variable is set to this value. This is only done if all datasets were
            labeled or refined. Defaults to None.

    Examples:

            deeplabcut.merge_datasets("/analysis/project/reaching-task/config.yaml")
    """

    cfg = auxiliaryfunctions.read_config(config)
    config_path = Path(config).parents[0]

    bf = config_path / "labeled-data"
    allfolders = [
        p for p in bf.iterdir() if "_labeled" not in p.name and not p.name.startswith(".")
    ]  # exclude labeled data folders and temporary files
    flagged = False
    for _findex, folder in enumerate(allfolders):
        if (folder / "MachineLabelsRefine.h5").is_file():  # Folder that was manually refine...
            pass
        elif (folder / ("CollectedData_" + cfg["scorer"] + ".h5")).is_file():  # Folder that contains human data set...
            pass
        else:
            print("The following folder was not manually refined,...", folder)
            flagged = True
            # this folder does not contain a MachineLabelsRefine file (not updated...)

    if not flagged:
        # updates iteration by 1
        iter_prev = cfg["iteration"]
        if not forceiterate:
            cfg["iteration"] = int(iter_prev + 1)
        else:
            cfg["iteration"] = forceiterate

        auxiliaryfunctions.write_config(config, cfg)

        print("Merged data sets and updated refinement iteration to " + str(cfg["iteration"]) + ".")
        print("Now you can create a new training set for the expanded annotated images (use create_training_dataset).")
    else:
        print("Please label, or remove the un-corrected folders.")