Skip to content

deeplabcut.modelzoo.generalized_data_converter.datasets.ma_dlc_dataframe

Functions:

Name Description
merge_annotateddatasets

Merges all the h5 files for all labeled-datasets (from individual videos).

merge_annotateddatasets 

merge_annotateddatasets(cfg)

Merges all the h5 files for all labeled-datasets (from individual videos).

This is a bit of a mess because of cross platform compatibility.

Within platform comp. is straightforward. But if someone labels on windows and wants to train on a unix cluster or colab...

Source code in deeplabcut/modelzoo/generalized_data_converter/datasets/ma_dlc_dataframe.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def merge_annotateddatasets(cfg):
    """Merges all the h5 files for all labeled-datasets (from individual videos).

    This is a bit of a mess because of cross platform compatibility.

    Within platform comp. is straightforward.
    But if someone labels on windows and wants to train on a unix cluster or colab...
    """
    AnnotationData = []
    data_path = Path(os.path.join(cfg["project_path"], "labeled-data"))
    videos = cfg["video_sets"].keys()
    video_filenames = parse_video_filenames(videos)
    for filename in video_filenames:
        file_path = os.path.join(data_path / filename, f"CollectedData_{cfg['scorer']}.h5")
        try:
            data = pd.read_hdf(file_path)
            conversioncode.guarantee_multiindex_rows(data)
            if data.columns.levels[0][0] != cfg["scorer"]:
                print(
                    f"{file_path} labeled by a different scorer. "
                    "This data will not be utilized in training dataset creation. "
                    "If you need to merge datasets across scorers, see "
                    "https://github.com/DeepLabCut/DeepLabCut/wiki/"
                    "Using-labeled-data-in-DeepLabCut-that-was-annotated-elsewhere-(or-merge-across-labelers)"
                )
                continue
            AnnotationData.append(data)
        except FileNotFoundError:
            print(file_path, " not found (perhaps not annotated).")

    if not len(AnnotationData):
        print(
            "Annotation data was not found by splitting video paths (from config['video_sets'])."
            " An alternative route is taken..."
        )
        AnnotationData = conversioncode.merge_windowsannotationdataONlinuxsystem(cfg)
        if not len(AnnotationData):
            print("No data was found!")
            return

    AnnotationData = pd.concat(AnnotationData).sort_index()
    # When concatenating DataFrames with misaligned column labels,
    # all sorts of reordering may happen (mainly depending on 'sort' and 'join')
    # Ensure the 'bodyparts' level agrees with the order in the config file.
    if cfg.get("multianimalproject", False):
        (
            _,
            uniquebodyparts,
            multianimalbodyparts,
        ) = auxfun_multianimal.extractindividualsandbodyparts(cfg)
        bodyparts = multianimalbodyparts + uniquebodyparts
    else:
        bodyparts = cfg["bodyparts"]
    AnnotationData = AnnotationData.reindex(bodyparts, axis=1, level=AnnotationData.columns.names.index("bodyparts"))
    AnnotationData = drop_likelihood_columns(AnnotationData)

    return AnnotationData