deeplabcut.utils.auxiliaryfunctions

DeepLabCut2.0 Toolbox (deeplabcut.org) © A. & M. Mathis Labs https://github.com/DeepLabCut/DeepLabCut Please see AUTHORS for contributors.

https://github.com/DeepLabCut/DeepLabCut/blob/master/AUTHORS Licensed under GNU Lesser General Public License v3.0

Functions:

Name	Description
attempt_to_make_folder	Attempts to create a folder with specified name.
check_if_post_processing	Checks if filtered/bone lengths were already calculated.
create_config_template	Creates a template for config.yaml file.
create_config_template_3d	Creates a template for config.yaml file for 3d project.
edit_config	Convenience function to edit and save a config file from a dictionary.
filter_files_by_patterns	Filters files in a folder based on start, contain, and end patterns.
find_analyzed_data	Find potential data files from the hints given to the function.
find_video_metadata	For backward compatibility, let us search the substring 'meta'.
get_bodyparts	Args:
get_deeplabcut_path	Get path of where deeplabcut is currently running.
get_evaluation_folder	Args:
get_immediate_subdirectories	Get list of immediate subdirectories.
get_model_folder	Args:
get_scorer_name	Extract the scorer/network name for a particular shuffle, training fraction, etc.
get_snapshots_from_folder	Returns an ordered list of existing snapshot names in the train folder, sorted by
get_training_set_folder	Training Set folder for config file based on parameters.
get_unique_bodyparts	Args:
get_video_list	Get list of videos in a path (if filetype == all), otherwise just a specific
grab_files_in_folder	Return the paths of files with extension ext present in folder.
intersection_of_body_parts_and_ones_given_by_user	Returns all body parts when comparisonbodyparts=='all', otherwise all bpts that
read_config	Reads structured config file defining a project.
read_pickle	Read the pickle file.
save_data	Save predicted data as h5 file and metadata as pickle file; created by
write_config	Write structured config file.
write_config_3d	Write structured 3D config file.
write_pickle	Write the pickle file.

attempt_to_make_folder

attempt_to_make_folder(foldername, recursive=False)

Attempts to create a folder with specified name.

Does nothing if it already exists.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def attempt_to_make_folder(foldername, recursive=False):
    """Attempts to create a folder with specified name.

    Does nothing if it already exists.
    """
    try:
        os.path.isdir(foldername)
    except TypeError:  # https://www.python.org/dev/peps/pep-0519/
        foldername = os.fspath(foldername)  # https://github.com/DeepLabCut/DeepLabCut/issues/105 (windows)

    if os.path.isdir(foldername):
        pass
    else:
        if recursive:
            os.makedirs(foldername)
        else:
            os.mkdir(foldername)

check_if_post_processing

check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix='filtered')

Checks if filtered/bone lengths were already calculated.

If not, figures out if data was already analyzed (either with legacy scorer name or new one!)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def check_if_post_processing(folder, vname, DLCscorer, DLCscorerlegacy, suffix="filtered"):
    """Checks if filtered/bone lengths were already calculated.

    If not, figures out if data was already analyzed (either with legacy scorer name or
    new one!)
    """
    outdataname = os.path.join(folder, vname + DLCscorer + suffix + ".h5")
    sourcedataname = os.path.join(folder, vname + DLCscorer + ".h5")
    if os.path.isfile(outdataname):  # was data already processed?
        if suffix == "filtered":
            print("Video already filtered...", outdataname)
        elif suffix == "_skeleton":
            print("Skeleton in video already processed...", outdataname)

        return False, outdataname, sourcedataname, DLCscorer
    else:
        odn = os.path.join(folder, vname + DLCscorerlegacy + suffix + ".h5")
        if os.path.isfile(odn):  # was it processed by DLC <2.1 project?
            if suffix == "filtered":
                print("Video already filtered...(with DLC<2.1)!", odn)
            elif suffix == "_skeleton":
                print("Skeleton in video already processed... (with DLC<2.1)!", odn)
            return False, odn, odn, DLCscorerlegacy
        else:
            sdn = os.path.join(folder, vname + DLCscorerlegacy + ".h5")
            tracks = sourcedataname.replace(".h5", "tracks.h5")
            if os.path.isfile(sourcedataname):  # Was the video already analyzed?
                return True, outdataname, sourcedataname, DLCscorer
            elif os.path.isfile(sdn):  # was it analyzed with DLC<2.1?
                return True, odn, sdn, DLCscorerlegacy
            elif os.path.isfile(tracks):  # May be a MA project with tracklets
                return True, tracks.replace(".h5", f"{suffix}.h5"), tracks, DLCscorer
            else:
                print("Video not analyzed -- Run analyze_videos first.")
                return False, outdataname, sourcedataname, DLCscorer

create_config_template

create_config_template(multianimal=False)

Creates a template for config.yaml file.

This specific order is preserved while saving as yaml file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def create_config_template(multianimal=False):
    """Creates a template for config.yaml file.

    This specific order is preserved while saving as yaml file.
    """
    if multianimal:
        yaml_str = """\
# Project definitions (do not edit)
Task:
scorer:
date:
multianimalproject:
identity:
\n
# Project path (change when moving around)
project_path:
\n
# Default DeepLabCut engine to use for shuffle creation (either pytorch or tensorflow)
engine: pytorch
\n
# Annotation data set configuration (and individual video cropping parameters)
video_sets:
individuals:
uniquebodyparts:
multianimalbodyparts:
bodyparts:
\n
# Fraction of video to start/stop when extracting frames for labeling/refinement
start:
stop:
numframes2pick:
\n
# Plotting configuration
skeleton:
skeleton_color:
pcutoff:
dotsize:
alphavalue:
colormap:
\n
# Training,Evaluation and Analysis configuration
TrainingFraction:
iteration:
default_net_type:
default_augmenter:
default_track_method:
snapshotindex:
detector_snapshotindex:
batch_size:
\n
# Cropping Parameters (for analysis and outlier frame detection)
cropping:
#if cropping is true for analysis, then set the values here:
x1:
x2:
y1:
y2:
\n
# Refinement configuration (parameters from annotation dataset configuration also relevant in this stage)
corner2move2:
move2corner:
\n
# Conversion tables to fine-tune SuperAnimal weights
SuperAnimalConversionTables:
        """
    else:
        yaml_str = """\
# Project definitions (do not edit)
Task:
scorer:
date:
multianimalproject:
identity:
\n
# Project path (change when moving around)
project_path:
\n
# Default DeepLabCut engine to use for shuffle creation (either pytorch or tensorflow)
engine: pytorch
\n
# Annotation data set configuration (and individual video cropping parameters)
video_sets:
bodyparts:
\n
# Fraction of video to start/stop when extracting frames for labeling/refinement
start:
stop:
numframes2pick:
\n
# Plotting configuration
skeleton:
skeleton_color:
pcutoff:
dotsize:
alphavalue:
colormap:
\n
# Training,Evaluation and Analysis configuration
TrainingFraction:
iteration:
default_net_type:
default_augmenter:
snapshotindex:
detector_snapshotindex:
batch_size:
detector_batch_size:
\n
# Cropping Parameters (for analysis and outlier frame detection)
cropping:
#if cropping is true for analysis, then set the values here:
x1:
x2:
y1:
y2:
\n
# Refinement configuration (parameters from annotation dataset configuration also relevant in this stage)
corner2move2:
move2corner:
\n
# Conversion tables to fine-tune SuperAnimal weights
SuperAnimalConversionTables:
        """

    ruamelFile = YAML()
    cfg_file = ruamelFile.load(yaml_str)
    return cfg_file, ruamelFile

create_config_template_3d

create_config_template_3d()

Creates a template for config.yaml file for 3d project.

This specific order is preserved while saving as yaml file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def create_config_template_3d():
    """Creates a template for config.yaml file for 3d project.

    This specific order is preserved while saving as yaml file.
    """
    yaml_str = """\
# Project definitions (do not edit)
Task:
scorer:
date:
\n
# Project path (change when moving around)
project_path:
\n
# Plotting configuration
skeleton: # Note that the pairs must be defined, as you want them linked!
skeleton_color:
pcutoff:
colormap:
dotsize:
alphaValue:
markerType:
markerColor:
\n
# Number of cameras, camera names, path of the config files, shuffle index and trainingsetindex used to analyze videos:
num_cameras:
camera_names:
scorername_3d: # Enter the scorer name for the 3D output
    """
    ruamelFile_3d = YAML()
    cfg_file_3d = ruamelFile_3d.load(yaml_str)
    return cfg_file_3d, ruamelFile_3d

edit_config

edit_config(configname, edits, output_name='')

Convenience function to edit and save a config file from a dictionary.

Parameters

configname : string String containing the full path of the config file in the project. edits : dict Key–value pairs to edit in config output_name : string, optional (default='') Overwrite the original config.yaml by default. If passed in though, new filename of the edited config.

Examples

config_path = 'my_stellar_lab/dlc/config.yaml'

edits = {'numframes2pick': 5, 'trainingFraction': [0.5, 0.8], 'skeleton': [['a', 'b'], ['b', 'c']]}

deeplabcut.auxiliaryfunctions.edit_config(config_path, edits)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def edit_config(configname, edits, output_name=""):
    """Convenience function to edit and save a config file from a dictionary.

    Parameters
    ----------
    configname : string
        String containing the full path of the config file in the project.
    edits : dict
        Key–value pairs to edit in config
    output_name : string, optional (default='')
        Overwrite the original config.yaml by default.
        If passed in though, new filename of the edited config.

    Examples
    --------
    config_path = 'my_stellar_lab/dlc/config.yaml'

    edits = {'numframes2pick': 5,
             'trainingFraction': [0.5, 0.8],
             'skeleton': [['a', 'b'], ['b', 'c']]}

    deeplabcut.auxiliaryfunctions.edit_config(config_path, edits)
    """
    cfg = read_plainconfig(configname)
    for key, value in edits.items():
        cfg[key] = value
    if not output_name:
        output_name = configname
    try:
        write_plainconfig(output_name, cfg)
    except ruamel.yaml.representer.RepresenterError:
        warnings.warn("Some edits could not be written. The configuration file will be left unchanged.", stacklevel=2)
        for key in edits:
            cfg.pop(key)
        write_plainconfig(output_name, cfg)
    return cfg

filter_files_by_patterns

filter_files_by_patterns(
    folder: str | Path,
    start_patterns: set[str] | None = None,
    contain_patterns: set[str] | None = None,
    end_patterns: set[str] | None = None,
) -> list[Path]

Filters files in a folder based on start, contain, and end patterns.

Parameters:

Name	Type	Description	Default
folder	str | Path	The folder to search for files.	required
start_patterns	Set[str] | None	Patterns the filenames should start with. If None or empty, this pattern is not taken into account.	None
contain_patterns	set[str]	Patterns the filenames should contain. If None or empty, this pattern is not taken into account.	None
end_patterns	set[str]	Patterns the filenames should end with. If None or empty, this pattern is not taken into account.	None

Returns:

Type	Description
list[Path]	List[Path]: List of files that match the criteria.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def filter_files_by_patterns(
    folder: str | Path,
    start_patterns: set[str] | None = None,
    contain_patterns: set[str] | None = None,
    end_patterns: set[str] | None = None,
) -> list[Path]:
    """Filters files in a folder based on start, contain, and end patterns.

    Args:
        folder (str | Path): The folder to search for files.

        start_patterns (Set[str] | None): Patterns the filenames should start with.
            If None or empty, this pattern is not taken into account.

        contain_patterns (set[str]): Patterns the filenames should contain.
            If None or empty, this pattern is not taken into account.

        end_patterns (set[str]): Patterns the filenames should end with.
            If None or empty, this pattern is not taken into account.

    Returns:
        List[Path]: List of files that match the criteria.
    """
    folder = Path(folder)  # Ensure the folder is a Path object
    if not folder.is_dir():
        raise ValueError(f"{folder} is not a valid directory.")

    # Filter files based on the given patterns
    matching_files = [
        file
        for file in folder.iterdir()
        if file.is_file()
        and (not start_patterns or any(file.name.startswith(start) for start in start_patterns))
        and (not contain_patterns or any(contain in file.name for contain in contain_patterns))
        and (not end_patterns or any(file.name.endswith(end) for end in end_patterns))
    ]

    return matching_files

find_analyzed_data

find_analyzed_data(folder, videoname: str, scorer: str, filtered=False, track_method='')

Find potential data files from the hints given to the function.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def find_analyzed_data(folder, videoname: str, scorer: str, filtered=False, track_method=""):
    """Find potential data files from the hints given to the function."""

    scorer_legacy = scorer.replace("DLC", "DeepCut")
    suffix = "_filtered" if filtered else ""
    tracker = TRACK_METHODS.get(track_method, "")

    candidates = []
    for file in grab_files_in_folder(folder, "h5"):
        stem = Path(file).stem.replace("_filtered", "")
        starts_by_scorer = file.startswith(videoname + scorer) or file.startswith(videoname + scorer_legacy)
        if tracker:
            matches_tracker = stem.endswith(tracker)
        else:
            matches_tracker = not any(stem.endswith(s) for s in TRACK_METHODS.values())
        if all(
            (
                starts_by_scorer,
                "skeleton" not in file,
                matches_tracker,
                (filtered and "filtered" in file) or (not filtered and "filtered" not in file),
            )
        ):
            candidates.append(file)

    if not len(candidates):
        msg = (
            f"No {'un' if not filtered else ''}filtered data file found in {folder} "
            f"for video {videoname} and scorer {scorer}"
        )
        if track_method:
            msg += f" and {track_method} tracker"
        msg += "."
        raise FileNotFoundError(msg)

    n_candidates = len(candidates)
    if n_candidates > 1:  # This should not be happening anyway...
        print(f"{n_candidates} possible data files were found: {candidates}.\nPicking the first by default...")
    filepath = str(Path(folder) / candidates[0])
    scorer = scorer if scorer in filepath else scorer_legacy
    return filepath, scorer, suffix

find_video_metadata

find_video_metadata(folder, videoname: str, scorer: str)

For backward compatibility, let us search the substring 'meta'.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def find_video_metadata(folder, videoname: str, scorer: str):
    """For backward compatibility, let us search the substring 'meta'."""

    scorer_legacy = scorer.replace("DLC", "DeepCut")
    meta_files = filter_files_by_patterns(
        folder=folder,
        start_patterns={videoname + scorer, videoname + scorer_legacy},
        contain_patterns={"meta"},
        end_patterns={"pickle"},
    )
    if not meta_files:
        raise FileNotFoundError(f"No metadata found in {folder} for video {videoname} and scorer {scorer}.")
    return meta_files[0]

get_bodyparts

get_bodyparts(cfg: dict) -> list[str]

Parameters:

Name	Type	Description	Default
cfg	dict	a project configuration file	required

Returns: bodyparts listed in the project (does not include the unique_bodyparts entry)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_bodyparts(cfg: dict) -> list[str]:
    """
    Args:
        cfg: a project configuration file

    Returns: bodyparts listed in the project (does not include the unique_bodyparts entry)
    """
    if cfg.get("multianimalproject", False):
        (
            _,
            _,
            multianimal_bodyparts,
        ) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
        return multianimal_bodyparts

    return cfg["bodyparts"]

get_deeplabcut_path

get_deeplabcut_path()

Get path of where deeplabcut is currently running.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_deeplabcut_path():
    """Get path of where deeplabcut is currently running."""
    import importlib.util

    return os.path.split(importlib.util.find_spec("deeplabcut").origin)[0]

get_evaluation_folder

get_evaluation_folder(
    trainFraction: float, shuffle: int, cfg: dict, engine: Engine | None = None, modelprefix: str = ""
) -> Path

Parameters:

Name	Type	Description	Default
trainFraction	float	the training fraction (as defined in the project configuration) for which to get the evaluation folder	required
shuffle	int	the index of the shuffle for which to get the evaluation folder	required
cfg	dict	the project configuration	required
engine	Engine | None	The engine for which we want the model folder. Defaults to None, which automatically gets the engine for the shuffle from the training dataset metadata file.	None
modelprefix	str	The name of the folder	''

Returns:

Type	Description
Path	the relative path from the project root to the folder containing the model files for a shuffle (configuration files, snapshots, training logs, ...)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_evaluation_folder(
    trainFraction: float,
    shuffle: int,
    cfg: dict,
    engine: Engine | None = None,
    modelprefix: str = "",
) -> Path:
    """
    Args:
        trainFraction: the training fraction (as defined in the project configuration)
            for which to get the evaluation folder
        shuffle: the index of the shuffle for which to get the evaluation folder
        cfg: the project configuration
        engine: The engine for which we want the model folder. Defaults to None,
            which automatically gets the engine for the shuffle from the training
            dataset metadata file.
        modelprefix: The name of the folder

    Returns:
        the relative path from the project root to the folder containing the model files
        for a shuffle (configuration files, snapshots, training logs, ...)
    """
    if engine is None:
        from deeplabcut.generate_training_dataset.metadata import get_shuffle_engine

        engine = get_shuffle_engine(
            cfg=cfg,
            trainingsetindex=cfg["TrainingFraction"].index(trainFraction),
            shuffle=shuffle,
            modelprefix=modelprefix,
        )

    Task = cfg["Task"]
    date = cfg["date"]
    iterate = "iteration-" + str(cfg["iteration"])
    if "eval_prefix" in cfg:
        eval_prefix = cfg["eval_prefix"]
    else:
        eval_prefix = engine.results_folder_name
    return Path(
        modelprefix,
        eval_prefix,
        iterate,
        Task + date + "-trainset" + str(int(trainFraction * 100)) + "shuffle" + str(shuffle),
    )

get_immediate_subdirectories

get_immediate_subdirectories(a_dir)

Get list of immediate subdirectories.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_immediate_subdirectories(a_dir):
    """Get list of immediate subdirectories."""
    return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))]

get_model_folder

get_model_folder(
    trainFraction: float, shuffle: int, cfg: dict, modelprefix: str = "", engine: Engine = Engine.TF
) -> Path

Parameters:

Name	Type	Description	Default
trainFraction	float	the training fraction (as defined in the project configuration) for which to get the model folder	required
shuffle	int	the index of the shuffle for which to get the model folder	required
cfg	dict	the project configuration	required
modelprefix	str	The name of the folder	''
engine	Engine	The engine for which we want the model folder. Defaults to tensorflow for backwards compatibility with DeepLabCut 2.X	TF

Returns:

Type	Description
Path	the relative path from the project root to the folder containing the model files for a shuffle (configuration files, snapshots, training logs, ...)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_model_folder(
    trainFraction: float,
    shuffle: int,
    cfg: dict,
    modelprefix: str = "",
    engine: Engine = Engine.TF,
) -> Path:
    """
    Args:
        trainFraction: the training fraction (as defined in the project configuration)
            for which to get the model folder
        shuffle: the index of the shuffle for which to get the model folder
        cfg: the project configuration
        modelprefix: The name of the folder
        engine: The engine for which we want the model folder. Defaults to `tensorflow`
            for backwards compatibility with DeepLabCut 2.X

    Returns:
        the relative path from the project root to the folder containing the model files
        for a shuffle (configuration files, snapshots, training logs, ...)
    """
    proj_id = f"{cfg['Task']}{cfg['date']}"
    return Path(
        modelprefix,
        engine.model_folder_name,
        f"iteration-{cfg['iteration']}",
        f"{proj_id}-trainset{int(trainFraction * 100)}shuffle{shuffle}",
    )

get_scorer_name

get_scorer_name(
    cfg: dict,
    shuffle: int,
    trainFraction: float,
    trainingsiterations: str | int = "unknown",
    modelprefix: str = "",
    engine: Engine | None = None,
    **kwargs
)

Extract the scorer/network name for a particular shuffle, training fraction, etc. If the engine is not specified, determines which to use from kwargs: additional arguments. For torch-based shuffles, can be used to specify: - snapshot_index - detector_snapshot_index

Returns tuple of DLCscorer, DLCscorerlegacy (old naming convention)

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_scorer_name(
    cfg: dict,
    shuffle: int,
    trainFraction: float,
    trainingsiterations: str | int = "unknown",
    modelprefix: str = "",
    engine: Engine | None = None,
    **kwargs,
):
    """Extract the scorer/network name for a particular shuffle, training fraction, etc.
    If the engine is not specified, determines which to use from
    kwargs: additional arguments.
        For torch-based shuffles, can be used to specify:
            - snapshot_index
            - detector_snapshot_index

    Returns tuple of DLCscorer, DLCscorerlegacy (old naming convention)
    """
    if engine is None:
        from deeplabcut.generate_training_dataset.metadata import get_shuffle_engine

        engine = get_shuffle_engine(
            cfg=cfg,
            trainingsetindex=cfg["TrainingFraction"].index(trainFraction),
            shuffle=shuffle,
            modelprefix=modelprefix,
        )

    if engine == Engine.PYTORCH:
        from deeplabcut.pose_estimation_pytorch.apis.utils import get_scorer_name

        snapshot_index = kwargs.get("snapshot_index", None)
        detector_snapshot_index = kwargs.get("detector_snapshot_index", None)
        dlc3_scorer = get_scorer_name(
            cfg=cfg,
            shuffle=shuffle,
            train_fraction=trainFraction,
            snapshot_index=snapshot_index,
            detector_index=detector_snapshot_index,
            modelprefix=modelprefix,
        )
        return dlc3_scorer, dlc3_scorer

    Task = cfg["Task"]
    date = cfg["date"]

    if trainingsiterations == "unknown":
        snapshotindex = get_snapshot_index_for_scorer("snapshotindex", cfg["snapshotindex"])
        model_folder = get_model_folder(trainFraction, shuffle, cfg, engine=engine, modelprefix=modelprefix)
        train_folder = Path(cfg["project_path"]) / model_folder / "train"
        snapshot_names = get_snapshots_from_folder(train_folder)
        snapshot_name = snapshot_names[snapshotindex]
        trainingsiterations = (snapshot_name.split(os.sep)[-1]).split("-")[-1]

    dlc_cfg = read_plainconfig(
        os.path.join(
            cfg["project_path"],
            str(get_model_folder(trainFraction, shuffle, cfg, engine=engine, modelprefix=modelprefix)),
            "train",
            engine.pose_cfg_name,
        )
    )
    # ABBREVIATE NETWORK NAMES -- esp. for mobilenet!
    if "resnet" in dlc_cfg["net_type"]:
        if dlc_cfg.get("multi_stage", False):
            netname = "dlcrnetms5"
        else:
            netname = dlc_cfg["net_type"].replace("_", "")
    elif "mobilenet" in dlc_cfg["net_type"]:  # mobilenet >> mobnet_100; mobnet_35 etc.
        netname = "mobnet_" + str(int(float(dlc_cfg["net_type"].split("_")[-1]) * 100))
    elif "efficientnet" in dlc_cfg["net_type"]:
        netname = "effnet_" + dlc_cfg["net_type"].split("-")[1]
    else:
        raise ValueError(f"Failed to abbreviate network name: {dlc_cfg['net_type']}")

    scorer = "DLC_" + netname + "_" + Task + str(date) + "shuffle" + str(shuffle) + "_" + str(trainingsiterations)
    # legacy scorername until DLC 2.1. (cfg['resnet'] is deprecated / which is why we get the resnet_xyz name from
    # dlc_cfg!
    # scorer_legacy = 'DeepCut' + "_resnet" + str(cfg['resnet']) + "_" + Task + str(date) + 'shuffle' + str(shuffle) +
    # '_' + str(trainingsiterations)
    scorer_legacy = scorer.replace("DLC", "DeepCut")
    return scorer, scorer_legacy

get_snapshots_from_folder

get_snapshots_from_folder(train_folder: Path) -> list[str]

Returns an ordered list of existing snapshot names in the train folder, sorted by increasing training iterations.

Raises:

Type	Description
FileNotFoundError	if no snapshot_names are found in the train_folder.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_snapshots_from_folder(train_folder: Path) -> list[str]:
    """Returns an ordered list of existing snapshot names in the train folder, sorted by
    increasing training iterations.

    Raises:
        FileNotFoundError: if no snapshot_names are found in the train_folder.
    """
    snapshot_names = [file.stem for file in train_folder.iterdir() if "index" in file.name]

    if len(snapshot_names) == 0:
        raise FileNotFoundError(
            f"No snapshots were found in {train_folder}! Please ensure the network has "
            f"been trained and verify the iteration, shuffle and trainFraction are "
            f"correct."
        )

    # sort in ascending order of iteration number
    return sorted(snapshot_names, key=lambda name: int(name.split("-")[1]))

get_training_set_folder

get_training_set_folder(cfg: dict) -> Path

Training Set folder for config file based on parameters.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_training_set_folder(cfg: dict) -> Path:
    """Training Set folder for config file based on parameters."""
    Task = cfg["Task"]
    date = cfg["date"]
    iterate = "iteration-" + str(cfg["iteration"])
    return Path(os.path.join("training-datasets", iterate, "UnaugmentedDataSet_" + Task + date))

get_unique_bodyparts

get_unique_bodyparts(cfg: dict) -> list[str]

Parameters:

Name	Type	Description	Default
cfg	dict	a project configuration file	required

Returns: all unique bodyparts listed in the project

Source code in deeplabcut/utils/auxiliaryfunctions.py

def get_unique_bodyparts(cfg: dict) -> list[str]:
    """
    Args:
        cfg: a project configuration file

    Returns: all unique bodyparts listed in the project
    """
    if cfg.get("multianimalproject", False):
        (
            _,
            unique_bodyparts,
            _,
        ) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
        return unique_bodyparts

    return []

get_video_list

get_video_list(filename, videopath, videtype)

Get list of videos in a path (if filetype == all), otherwise just a specific file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

@deprecated(replacement="deeplabcut.collect_video_paths", since="3.0.0")
def get_video_list(filename, videopath, videtype):
    """Get list of videos in a path (if filetype == all), otherwise just a specific
    file."""
    videos = list(grab_files_in_folder(videopath, videtype))
    if filename == "all":
        return videos
    else:
        if filename in videos:
            videos = [filename]
        else:
            videos = []
            print("Video not found!", filename)
    return videos

grab_files_in_folder

grab_files_in_folder(folder, ext='', relative=True)

Return the paths of files with extension ext present in folder.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def grab_files_in_folder(folder, ext="", relative=True):
    """Return the paths of files with extension *ext* present in *folder*."""
    for file in os.listdir(folder):
        if file.endswith(ext):
            yield file if relative else os.path.join(folder, file)

intersection_of_body_parts_and_ones_given_by_user

intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts)

Returns all body parts when comparisonbodyparts=='all', otherwise all bpts that are in the intersection of comparisonbodyparts and the actual bodyparts.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts):
    """Returns all body parts when comparisonbodyparts=='all', otherwise all bpts that
    are in the intersection of comparisonbodyparts and the actual bodyparts."""
    # if "MULTI!" in allbpts:
    if cfg["multianimalproject"]:
        allbpts = cfg["multianimalbodyparts"] + cfg["uniquebodyparts"]
    else:
        allbpts = cfg["bodyparts"]

    if comparisonbodyparts == "all":
        return list(allbpts)
    else:  # take only items in list that are actually bodyparts...
        cpbpts = [bp for bp in allbpts if bp in comparisonbodyparts]
        # Ensure same order as in config.yaml
        return cpbpts

read_config

read_config(configname)

Reads structured config file defining a project.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def read_config(configname):
    """Reads structured config file defining a project."""
    ruamelFile = YAML()
    path = Path(configname)
    if os.path.exists(path):
        try:
            with open(path) as f:
                cfg = ruamelFile.load(f)
                curr_dir = str(Path(configname).parent.resolve())

                if cfg.get("engine") is None:
                    cfg["engine"] = Engine.TF.aliases[0]
                    write_config(configname, cfg)

                if cfg.get("detector_snapshotindex") is None:
                    cfg["detector_snapshotindex"] = -1

                if cfg.get("detector_batch_size") is None:
                    cfg["detector_batch_size"] = 1

                if cfg["project_path"] != curr_dir:
                    cfg["project_path"] = curr_dir
                    write_config(configname, cfg)
        except Exception as err:
            if len(err.args) > 2:
                if err.args[2] == "could not determine a constructor for the tag '!!python/tuple'":
                    with open(path) as ymlfile:
                        cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader)
                        write_config(configname, cfg)
                else:
                    raise

    else:
        raise FileNotFoundError(
            f"Config file at {path} not found. Please make sure that the file exists and/or that you passed the path of"
            f"the config file correctly!"
        )
    return cfg

read_pickle

read_pickle(filename)

Read the pickle file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def read_pickle(filename):
    """Read the pickle file."""
    with open(filename, "rb") as handle:
        return pickle.load(handle)

save_data

save_data(PredicteData, metadata, dataname, pdindex, imagenames, save_as_csv)

Save predicted data as h5 file and metadata as pickle file; created by predict_videos.py.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def save_data(PredicteData, metadata, dataname, pdindex, imagenames, save_as_csv):
    """Save predicted data as h5 file and metadata as pickle file; created by
    predict_videos.py."""
    DataMachine = pd.DataFrame(PredicteData, columns=pdindex, index=imagenames)
    if save_as_csv:
        print("Saving csv poses!")
        DataMachine.to_csv(dataname.split(".h5")[0] + ".csv")
    DataMachine.to_hdf(dataname, key="df_with_missing", format="table", mode="w")
    with open(dataname.split(".h5")[0] + "_meta.pickle", "wb") as f:
        # Pickle the 'data' dictionary using the highest protocol available.
        pickle.dump(metadata, f, pickle.HIGHEST_PROTOCOL)

write_config

write_config(configname, cfg)

Write structured config file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def write_config(configname, cfg):
    """Write structured config file."""
    with open(configname, "w") as cf:
        cfg_file, ruamelFile = create_config_template(cfg.get("multianimalproject", False))
        for key in cfg.keys():
            cfg_file[key] = cfg[key]

        # Adding default value for variable skeleton and skeleton_color for backward compatibility.
        if "skeleton" not in cfg.keys():
            cfg_file["skeleton"] = []
            cfg_file["skeleton_color"] = "black"
        # Use a very large width so long strings (e.g., file paths or keys with spaces)
        # are kept on a single line instead of being wrapped, which can otherwise cause
        # them to be emitted as complex keys. See also:
        # https://stackoverflow.com/questions/31197268/pyyaml-yaml-dump-produces-complex-key-for-string-key-122-chars/31199123#31199123
        ruamelFile.width = 1_000_000
        ruamelFile.dump(cfg_file, cf)

write_config_3d

write_config_3d(configname, cfg)

Write structured 3D config file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def write_config_3d(configname, cfg):
    """Write structured 3D config file."""
    with open(configname, "w") as cf:
        cfg_file, ruamelFile = create_config_template_3d()
        for key in cfg.keys():
            cfg_file[key] = cfg[key]
        ruamelFile.dump(cfg_file, cf)

write_pickle

write_pickle(filename, data)

Write the pickle file.

Source code in deeplabcut/utils/auxiliaryfunctions.py

def write_pickle(filename, data):
    """Write the pickle file."""
    with open(filename, "wb") as handle:
        pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL)