deeplabcut.pose_estimation_tensorflow.visualizemaps

Functions:

Name	Description
extract_maps	Extracts the scoremap, locref, partaffinityfields (if available).
extract_save_all_maps	Extract scoremap, location refinement field and part affinity field predictions.

extract_maps

extract_maps(config, shuffle=0, trainingsetindex=0, gputouse=None, rescale=False, Indices=None, modelprefix='')

Extracts the scoremap, locref, partaffinityfields (if available).

Parameters:

Name	Type	Description	Default
config	string	Full path of the config.yaml file as a string.	required
shuffle	integer	Integer specifying shuffle index of the training dataset. Defaults to 0.	0
trainingsetindex	int	Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". Defaults to 0.	0
gputouse	int	GPU index (see nvidia-smi). Use None if no GPU. Defaults to None.	None
rescale	bool	Evaluate the model at the 'global_scale' variable (as set in the test/pose_config.yaml file for a particular project). I.e. every image will be resized according to that scale and prediction will be compared to the resized ground truth. The error will be reported in pixels at rescaled to the original size. I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!. The evaluation images are also shown for the original size! Defaults to False.	False
Indices	list	Image indices for which to extract maps. Defaults to None.	None
modelprefix	str	Directory containing the deeplabcut models to use. Defaults to "".	''

Returns:

Name	Type	Description
dict		Dictionary indexed by trainingsetfraction, snapshotindex, and imageindex; each item contains (image, scmap, locref, paf, bpt names, partaffinity graph, imagename, True/False if this image was in trainingset).

Examples:

If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0:

deeplabcut.extract_maps(configfile, 0, Indices=[0, 103])

Source code in deeplabcut/pose_estimation_tensorflow/visualizemaps.py

def extract_maps(
    config,
    shuffle=0,
    trainingsetindex=0,
    gputouse=None,
    rescale=False,
    Indices=None,
    modelprefix="",
):
    """Extracts the scoremap, locref, partaffinityfields (if available).

    Args:
        config (string): Full path of the config.yaml file as a string.
        shuffle (integer): Integer specifying shuffle index of the training dataset. Defaults to 0.
        trainingsetindex (int, optional): Integer specifying which TrainingsetFraction to use. By default the first
            (note that TrainingFraction is a list in config.yaml).
            This variable can also be set to "all". Defaults to 0.
        gputouse (int, optional): GPU index (see nvidia-smi). Use None if no GPU.
            Defaults to None.
        rescale (bool, optional): Evaluate the model at the 'global_scale' variable
            (as set in the test/pose_config.yaml file for a particular project).
            I.e. every image will be resized according to that scale
            and prediction will be compared to the resized ground truth.
            The error will be reported in pixels at rescaled to the *original* size.
            I.e. For a [200,200] pixel image evaluated at global_scale=.5, the predictions are calculated
            on [100,100] pixel images, compared to 1/2*ground truth and this error is then multiplied by 2!.
            The evaluation images are also shown for the original size! Defaults to False.
        Indices (list, optional): Image indices for which to extract maps. Defaults to None.
        modelprefix (str, optional): Directory containing the deeplabcut models to use.
            Defaults to "".

    Returns:
        dict: Dictionary indexed by trainingsetfraction, snapshotindex, and imageindex;
            each item contains (image, scmap, locref, paf, bpt names, partaffinity graph,
            imagename, True/False if this image was in trainingset).

    Examples:
        If you want to extract the data for image 0 and 103 (of the training set) for model trained with shuffle 0:

            deeplabcut.extract_maps(configfile, 0, Indices=[0, 103])
    """

    import numpy as np
    import pandas as pd
    import tensorflow as tf
    from tqdm import tqdm

    from deeplabcut.pose_estimation_tensorflow.config import load_config
    from deeplabcut.pose_estimation_tensorflow.core import (
        predict,
    )
    from deeplabcut.pose_estimation_tensorflow.core import (
        predict_multianimal as predictma,
    )
    from deeplabcut.pose_estimation_tensorflow.datasets.utils import data_to_input
    from deeplabcut.utils import auxiliaryfunctions
    from deeplabcut.utils.auxfun_videos import imread, imresize

    tf.compat.v1.reset_default_graph()
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"  #
    #    tf.logging.set_verbosity(tf.logging.WARN)

    start_path = Path.cwd()
    # Read file path for pose_config file. >> pass it on
    cfg = auxiliaryfunctions.read_config(config)

    if gputouse is not None:  # gpu selectinon
        os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)

    if trainingsetindex == "all":
        TrainingFractions = cfg["TrainingFraction"]
    else:
        if trainingsetindex < len(cfg["TrainingFraction"]) and trainingsetindex >= 0:
            TrainingFractions = [cfg["TrainingFraction"][int(trainingsetindex)]]
        else:
            raise Exception(
                "Please check the trainingsetindex! ",
                trainingsetindex,
                " should be an integer from 0 .. ",
                int(len(cfg["TrainingFraction"]) - 1),
            )

    # Loading human annotatated data
    trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg)
    Data = pd.read_hdf(Path(cfg["project_path"]) / str(trainingsetfolder) / ("CollectedData_" + cfg["scorer"] + ".h5"))

    # Make folder for evaluation
    auxiliaryfunctions.attempt_to_make_folder(Path(cfg["project_path"]) / "evaluation-results")

    Maps = {}
    for trainFraction in TrainingFractions:
        Maps[trainFraction] = {}
        ##################################################
        # Load and setup CNN part detector
        ##################################################
        datafn, metadatafn = auxiliaryfunctions.get_data_and_metadata_filenames(
            trainingsetfolder, trainFraction, shuffle, cfg
        )

        modelfolder = Path(cfg["project_path"]) / str(
            auxiliaryfunctions.get_model_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)
        )
        path_test_config = Path(modelfolder) / "test" / "pose_cfg.yaml"
        # Load meta data
        (
            data,
            trainIndices,
            testIndices,
            trainFraction,
        ) = auxiliaryfunctions.load_metadata(Path(cfg["project_path"]) / metadatafn)
        try:
            dlc_cfg = load_config(str(path_test_config))
        except FileNotFoundError as e:
            raise FileNotFoundError(
                f"It seems the model for shuffle {shuffle} and trainFraction {trainFraction} does not exist."
            ) from e

        # change batch size, if it was edited during analysis!
        dlc_cfg["batch_size"] = 1  # in case this was edited for analysis.

        # Create folder structure to store results.
        evaluationfolder = Path(cfg["project_path"]) / str(
            auxiliaryfunctions.get_evaluation_folder(trainFraction, shuffle, cfg, modelprefix=modelprefix)
        )
        auxiliaryfunctions.attempt_to_make_folder(evaluationfolder, recursive=True)

        Snapshots = auxiliaryfunctions.get_snapshots_from_folder(
            train_folder=Path(modelfolder) / "train",
        )

        if cfg["snapshotindex"] == -1:
            snapindices = [-1]
        elif cfg["snapshotindex"] == "all":
            snapindices = range(len(Snapshots))
        elif cfg["snapshotindex"] < len(Snapshots):
            snapindices = [cfg["snapshotindex"]]
        else:
            print("Invalid choice, only -1 (last), any integer up to last, or all (as string)!")

        ########################### RESCALING (to global scale)
        scale = dlc_cfg["global_scale"] if rescale else 1
        Data *= scale

        bptnames = [dlc_cfg["all_joints_names"][i] for i in range(len(dlc_cfg["all_joints"]))]

        for snapindex in snapindices:
            dlc_cfg["init_weights"] = str(
                Path(modelfolder) / "train" / Snapshots[snapindex]
            )  # setting weights to corresponding snapshot.
            Path(dlc_cfg["init_weights"]).name.split("-")[-1]  # read how many training siterations that corresponds to.

            # Name for deeplabcut net (based on its parameters)
            # DLCscorer,DLCscorerlegacy =
            # auxiliaryfunctions.GetScorerName(cfg,shuffle,trainFraction,trainingsiterations)
            # notanalyzed, resultsfilename,
            # DLCscorer=auxiliaryfunctions.CheckifNotEvaluated(str(evaluationfolder),
            # DLCscorer,DLCscorerlegacy,Snapshots[snapindex])
            # print("Extracting maps for ", DLCscorer, " with # of trainingiterations:", trainingsiterations)
            # if notanalyzed: #this only applies to ask if h5 exists...

            # Specifying state of model (snapshot / training state)
            sess, inputs, outputs = predict.setup_pose_prediction(dlc_cfg)
            Numimages = len(Data.index)
            np.zeros((Numimages, 3 * len(dlc_cfg["all_joints_names"])))
            print("Analyzing data...")
            if Indices is None:
                Indices = enumerate(Data.index)
            else:
                Ind = [Data.index[j] for j in Indices]
                Indices = enumerate(Ind)

            DATA = {}
            for imageindex, imagename in tqdm(Indices):
                image = imread(Path(cfg["project_path"]).joinpath(*imagename), mode="skimage")

                if scale != 1:
                    image = imresize(image, scale)

                image_batch = data_to_input(image)

                # Compute prediction with the CNN
                outputs_np = sess.run(outputs, feed_dict={inputs: image_batch})

                if cfg.get("multianimalproject", False):
                    scmap, locref, paf = predictma.extract_cnn_output(outputs_np, dlc_cfg)
                    pagraph = dlc_cfg["partaffinityfield_graph"]
                else:
                    scmap, locref = predict.extract_cnn_output(outputs_np, dlc_cfg)
                    paf = None
                    pagraph = []
                peaks = outputs_np[-1]

                if imageindex in testIndices:
                    trainingfram = False
                else:
                    trainingfram = True

                DATA[imageindex] = [
                    image,
                    scmap,
                    locref,
                    paf,
                    peaks,
                    bptnames,
                    pagraph,
                    imagename,
                    trainingfram,
                ]
            Maps[trainFraction][Snapshots[snapindex]] = DATA
    os.chdir(str(start_path))
    return Maps

extract_save_all_maps

extract_save_all_maps(
    config,
    shuffle=1,
    trainingsetindex=0,
    comparisonbodyparts="all",
    extract_paf=True,
    all_paf_in_one=True,
    gputouse=None,
    rescale=False,
    Indices=None,
    modelprefix="",
    dest_folder=None,
)

Extract scoremap, location refinement field and part affinity field predictions.

Maps are rescaled to the size of the input image and stored in the corresponding model folder in /evaluation-results.

Parameters:

Name	Type	Description	Default
config	string	Full path of the config.yaml file as a string.	required
shuffle	integer	Integer specifying shuffle index of the training dataset. Defaults to 1.	1
trainingsetindex	int	Integer specifying which TrainingsetFraction to use. By default the first (note that TrainingFraction is a list in config.yaml). This variable can also be set to "all". Defaults to 0.	0
comparisonbodyparts	list of bodyparts	Average error for those body parts only (subset of all body parts). Defaults to "all".	'all'
extract_paf	bool	Extract part affinity fields by default. Note that turning it off will make the function much faster. Defaults to True.	True
all_paf_in_one	bool	By default, all part affinity fields are displayed on a single frame. If false, individual fields are shown on separate frames. Defaults to True.	True
gputouse	int	GPU index (see nvidia-smi). Use None if no GPU. Defaults to None.	None
rescale	bool	Evaluate at global_scale from pose_config.yaml. Defaults to False.	False
Indices	list	Image indices for which to compute scmap/locref/paf. Defaults to None.	None
modelprefix	str	Directory containing the deeplabcut models to use. Defaults to "".	''
dest_folder	string	Destination folder for saved maps. Defaults to None.	None

Examples:

Calculated maps for images 0, 1 and 33.

deeplabcut.extract_save_all_maps(
    "/analysis/project/reaching-task/config.yaml", shuffle=1, Indices=[0, 1, 33]
)

Source code in deeplabcut/pose_estimation_tensorflow/visualizemaps.py

def extract_save_all_maps(
    config,
    shuffle=1,
    trainingsetindex=0,
    comparisonbodyparts="all",
    extract_paf=True,
    all_paf_in_one=True,
    gputouse=None,
    rescale=False,
    Indices=None,
    modelprefix="",
    dest_folder=None,
):
    """Extract scoremap, location refinement field and part affinity field predictions.

    Maps are rescaled to the size of the input image and stored in the corresponding
    model folder in /evaluation-results.

    Args:
        config (string): Full path of the config.yaml file as a string.
        shuffle (integer): Integer specifying shuffle index of the training dataset. Defaults to 1.
        trainingsetindex (int, optional): Integer specifying which TrainingsetFraction to use.
            By default the first (note that TrainingFraction is a list in config.yaml).
            This variable can also be set to "all". Defaults to 0.
        comparisonbodyparts (list of bodyparts): Average error for those body parts only
            (subset of all body parts). Defaults to "all".
        extract_paf (bool, optional): Extract part affinity fields by default.
            Note that turning it off will make the function much faster. Defaults to True.
        all_paf_in_one (bool, optional): By default, all part affinity fields are displayed on a single frame.
            If false, individual fields are shown on separate frames. Defaults to True.
        gputouse (int, optional): GPU index (see nvidia-smi). Use None if no GPU.
            Defaults to None.
        rescale (bool, optional): Evaluate at global_scale from pose_config.yaml.
            Defaults to False.
        Indices (list, optional): Image indices for which to compute scmap/locref/paf.
            Defaults to None.
        modelprefix (str, optional): Directory containing the deeplabcut models to use.
            Defaults to "".
        dest_folder (string, optional): Destination folder for saved maps. Defaults to None.

    Examples:
        Calculated maps for images 0, 1 and 33.

            deeplabcut.extract_save_all_maps(
                "/analysis/project/reaching-task/config.yaml", shuffle=1, Indices=[0, 1, 33]
            )

    """
    from tqdm import tqdm

    from deeplabcut.utils.auxiliaryfunctions import (
        attempt_to_make_folder,
        get_evaluation_folder,
        intersection_of_body_parts_and_ones_given_by_user,
        read_config,
    )

    cfg = read_config(config)
    data = extract_maps(config, shuffle, trainingsetindex, gputouse, rescale, Indices, modelprefix)

    comparisonbodyparts = intersection_of_body_parts_and_ones_given_by_user(cfg, comparisonbodyparts)

    print("Saving plots...")
    for frac, values in data.items():
        if not dest_folder:
            dest_folder = (
                Path(cfg["project_path"])
                / str(get_evaluation_folder(frac, shuffle, cfg, modelprefix=modelprefix))
                / "maps"
            )
        attempt_to_make_folder(dest_folder)
        filepath = "{imname}_{map}_{label}_{shuffle}_{frac}_{snap}.png"
        dest_path = str(Path(dest_folder) / filepath)
        for snap, maps in values.items():
            for imagenr in tqdm(maps):
                (
                    image,
                    scmap,
                    locref,
                    paf,
                    peaks,
                    bptnames,
                    pafgraph,
                    impath,
                    trainingframe,
                ) = maps[imagenr]
                if not extract_paf:
                    paf = None
                label = "train" if trainingframe else "test"
                imname = impath[-1]
                scmap, (locref_x, locref_y), paf = resize_all_maps(image, scmap, locref, paf)
                to_plot = [i for i, bpt in enumerate(bptnames) if bpt in comparisonbodyparts]
                list_of_inds = []
                for n, edge in enumerate(pafgraph):
                    if any(ind in to_plot for ind in edge):
                        list_of_inds.append([(2 * n, 2 * n + 1), (bptnames[edge[0]], bptnames[edge[1]])])
                if len(to_plot) > 1:
                    map_ = scmap[:, :, to_plot].sum(axis=2)
                    locref_x_ = locref_x[:, :, to_plot].sum(axis=2)
                    locref_y_ = locref_y[:, :, to_plot].sum(axis=2)
                elif len(to_plot) == 1 and len(bptnames) > 1:
                    map_ = scmap[:, :, to_plot]
                    locref_x_ = locref_x[:, :, to_plot]
                    locref_y_ = locref_y[:, :, to_plot]
                else:
                    map_ = scmap[..., 0]
                    locref_x_ = locref_x[..., 0]
                    locref_y_ = locref_y[..., 0]
                fig1, _ = visualize_scoremaps(image, map_)
                temp = dest_path.format(
                    imname=imname,
                    map="scmap",
                    label=label,
                    shuffle=shuffle,
                    frac=frac,
                    snap=snap,
                )
                fig1.savefig(temp)

                fig2, _ = visualize_locrefs(image, map_, locref_x_, locref_y_)
                temp = dest_path.format(
                    imname=imname,
                    map="locref",
                    label=label,
                    shuffle=shuffle,
                    frac=frac,
                    snap=snap,
                )
                fig2.savefig(temp)

                if paf is not None:
                    if not all_paf_in_one:
                        for inds, names in list_of_inds:
                            fig3, _ = visualize_paf(image, paf[:, :, [inds]])
                            temp = dest_path.format(
                                imname=imname,
                                map=f"paf_{'_'.join(names)}",
                                label=label,
                                shuffle=shuffle,
                                frac=frac,
                                snap=snap,
                            )
                            fig3.savefig(temp)
                    else:
                        inds = [elem[0] for elem in list_of_inds]
                        n_inds = len(inds)
                        cmap = plt.cm.get_cmap(cfg["colormap"], n_inds)
                        colors = cmap(range(n_inds))
                        fig3, _ = visualize_paf(image, paf[:, :, inds], colors=colors)
                        temp = dest_path.format(
                            imname=imname,
                            map="paf",
                            label=label,
                            shuffle=shuffle,
                            frac=frac,
                            snap=snap,
                        )
                        fig3.savefig(temp)
                plt.close("all")