Skip to content

deeplabcut.pose_estimation_tensorflow.modelzoo.api.spatiotemporal_adapt

Classes:

Name Description
SpatiotemporalAdaptation

SpatiotemporalAdaptation

Methods:

Name Description
__init__

Support video adaptation to a super model.
adaptation_training

There should be two choices, either taking a config, with is then assuming
Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
class SpatiotemporalAdaptation:
    @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
    def __init__(
        self,
        video_path,
        supermodel_name,
        scale_list=None,
        video_extensions: str | Sequence[str] | None = "mp4",
        adapt_iterations=1000,
        modelfolder="",
        customized_pose_config="",
        init_weights="",
    ):
        """Support video adaptation to a super model.

        Args:
            video_path (string): The string to the path of the video.
            init_weights (string): The path to a superanimal model's checkpoint.
            supermodel_name (string): Currently we support supertopview (LabMice) and
                superquadruped (quadruped side-view animals).
            scale_list (list): A list of different resolutions for the spatial pyramid.
            video_extensions (string or Sequence[str], optional): When the input is a directory, only videos with
                these extensions are analyzed. Defaults to ``"mp4"``.
            adapt_iterations (int): Number of iterations for adaptation training.
                Empirically 1000 is sufficient. Training longer can cause worse
                performance depending whether there is occlusion in the video.
            modelfolder (string, optional): Because the API does not need a dlc project,
                the checkpoint and logs go to this temporary model folder, and otherwise
                model is saved to the current work place.
            customized_pose_config (string, optional): Path to a custom pose config for
                non-modelzoo models. Defaults to "".

        Examples:
            Create a SpatiotemporalAdaptation object and perform inference, adaptation, and post-adaptation inference:

                from deeplabcut.pose_estimation_tensorflow.modelzoo.api.spatiotemporal_adapt import (
                    SpatiotemporalAdaptation,
                )
                video_path = "/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4"
                supermodel_name = "superanimal_topviewmouse"
                video_extensions = "mp4"
                adapter = SpatiotemporalAdaptation(
                    video_path,
                    supermodel_name,
                    modelfolder="temp_topview",
                    video_extensions=video_extensions,
                )
                adapter.before_adapt_inference()
                adapter.adaptation_training()
                adapter.after_adapt_inference()
        """
        if scale_list is None:
            scale_list = []

        self.video_path = video_path
        self.supermodel_name = supermodel_name
        self.scale_list = scale_list
        self.video_extensions = video_extensions
        vname = str(Path(self.video_path).stem)
        self.adapt_modelprefix = vname + "_video_adaptation"
        self.adapt_iterations = adapt_iterations
        self.modelfolder = modelfolder
        self.init_weights = init_weights

        project_name = "_".join(supermodel_name.split("_")[:-1])
        model_name = supermodel_name.split("_")[-1]
        self.project_name = project_name
        self.model_name = model_name

        if not customized_pose_config:
            dlc_root_path = get_deeplabcut_path()

            project_config = read_config(dlc_root_path / "modelzoo" / "project_configs" / f"{project_name}.yaml")

            model_config = read_config(dlc_root_path / "modelzoo" / "model_configs" / f"{model_name}.yaml")

            joints = [i for i in range(len(project_config["bodyparts"]))]
            num_joints = len(joints)
            model_config["all_joints"] = joints
            model_config["all_joints_names"] = project_config["bodyparts"]
            model_config["num_joints"] = num_joints
            model_config["num_limbs"] = int((num_joints * (num_joints - 1)) // 2)
            self.customized_pose_config = {**project_config, **model_config}
        else:
            self.customized_pose_config = customized_pose_config

    def before_adapt_inference(self, make_video=False, **kwargs):
        if self.init_weights != "":
            print("using customized weights", self.init_weights)
            _, datafiles = video_inference(
                [self.video_path],
                self.project_name,
                self.model_name,
                video_extensions=self.video_extensions,
                scale_list=self.scale_list,
                init_weights=self.init_weights,
                customized_test_config=self.customized_pose_config,
            )
        else:
            self.init_weights, datafiles = video_inference(
                [self.video_path],
                self.project_name,
                self.model_name,
                video_extensions=self.video_extensions,
                scale_list=self.scale_list,
                customized_test_config=self.customized_pose_config,
            )
        if kwargs.pop("plot_trajectories", True):
            if len(datafiles) == 0:
                print("No data files found for plotting trajectory")
            else:
                _plot_trajectories(datafiles[0])

        if make_video:
            create_labeled_video(
                "",
                [self.video_path],
                video_extensions=self.video_extensions,
                filtered=False,
                init_weights=self.init_weights,
                draw_skeleton=True,
                superanimal_name=self.project_name,
                **kwargs,
            )

    def train_without_project(self, pseudo_label_path, **kwargs):
        from deeplabcut.pose_estimation_tensorflow.core.train_multianimal import train

        displayiters = kwargs.pop("displayiters", 500)
        saveiters = kwargs.pop("saveiters", 1000)
        self.adapt_iterations = kwargs.pop("adapt_iterations", self.adapt_iterations)

        train(
            self.customized_pose_config,
            displayiters=displayiters,
            saveiters=saveiters,
            maxiters=self.adapt_iterations,
            modelfolder=self.modelfolder,
            init_weights=self.init_weights,
            pseudo_labels=pseudo_label_path,
            video_path=self.video_path,
            superanimal=self.supermodel_name,
            **kwargs,
        )

    def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs):
        """There should be two choices, either taking a config, with is then assuming
        there is a DLC project.

        Or we make up a fake one, then we use a light way convention to do adaptation
        """
        # looking for the pseudo label path
        DLCscorer = "DLC_" + Path(self.init_weights).stem
        vname = str(Path(self.video_path).stem)
        video_root = Path(self.video_path).parent

        _, pseudo_label_path, _, _ = load_analyzed_data(video_root, vname, DLCscorer, False, "")
        if self.modelfolder != "":
            Path(self.modelfolder).mkdir(parents=True, exist_ok=True)

        self.adapt_iterations = kwargs.get("adapt_iterations", self.adapt_iterations)

        self.train_without_project(
            pseudo_label_path,
            displayiters=displayiters,
            saveiters=saveiters,
            **kwargs,
        )

    def after_adapt_inference(self, create_labeled_video, **kwargs):
        ref_proj_config_path = ""

        files = list(Path(self.modelfolder).glob(f"snapshot-{self.adapt_iterations}.index"))

        if not len(files):
            raise ValueError("Weights were not found.")

        adapt_weights = files[0].replace(".index", "")

        # spatial pyramid is not for adapted model

        scale_list = kwargs.pop("scale_list", [])

        # spatial pyramid can still be useful for reducing jittering and quantization error

        _, datafiles = video_inference(
            [self.video_path],
            self.project_name,
            self.model_name,
            video_extensions=self.video_extensions,
            init_weights=adapt_weights,
            scale_list=scale_list,
            customized_test_config=self.customized_pose_config,
        )

        if kwargs.pop("plot_trajectories", True):
            _plot_trajectories(datafiles[0])

        if create_labeled_video:
            create_labeled_video(
                ref_proj_config_path,
                [self.video_path],
                video_extensions=self.video_extensions,
                filtered=False,
                init_weights=adapt_weights,
                draw_skeleton=True,
                superanimal_name=self.project_name,
                **kwargs,
            )

__init__ 

__init__(
    video_path,
    supermodel_name,
    scale_list=None,
    video_extensions: str | Sequence[str] | None = "mp4",
    adapt_iterations=1000,
    modelfolder="",
    customized_pose_config="",
    init_weights="",
)

Support video adaptation to a super model.

Parameters:

Name Type Description Default

video_path

string

The string to the path of the video.

 required

init_weights

string

The path to a superanimal model's checkpoint.

 ''

supermodel_name

string

Currently we support supertopview (LabMice) and superquadruped (quadruped side-view animals).

 required

scale_list

list

A list of different resolutions for the spatial pyramid.

 None

video_extensions

string or Sequence[str]

When the input is a directory, only videos with these extensions are analyzed. Defaults to "mp4".

 'mp4'

adapt_iterations

int

Number of iterations for adaptation training. Empirically 1000 is sufficient. Training longer can cause worse performance depending whether there is occlusion in the video.

 1000

modelfolder

string

Because the API does not need a dlc project, the checkpoint and logs go to this temporary model folder, and otherwise model is saved to the current work place.

 ''

customized_pose_config

string

Path to a custom pose config for non-modelzoo models. Defaults to "".

 ''

Examples:

Create a SpatiotemporalAdaptation object and perform inference, adaptation, and post-adaptation inference:

from deeplabcut.pose_estimation_tensorflow.modelzoo.api.spatiotemporal_adapt import (
    SpatiotemporalAdaptation,
)
video_path = "/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4"
supermodel_name = "superanimal_topviewmouse"
video_extensions = "mp4"
adapter = SpatiotemporalAdaptation(
    video_path,
    supermodel_name,
    modelfolder="temp_topview",
    video_extensions=video_extensions,
)
adapter.before_adapt_inference()
adapter.adaptation_training()
adapter.after_adapt_inference()
Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
@renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
def __init__(
    self,
    video_path,
    supermodel_name,
    scale_list=None,
    video_extensions: str | Sequence[str] | None = "mp4",
    adapt_iterations=1000,
    modelfolder="",
    customized_pose_config="",
    init_weights="",
):
    """Support video adaptation to a super model.

    Args:
        video_path (string): The string to the path of the video.
        init_weights (string): The path to a superanimal model's checkpoint.
        supermodel_name (string): Currently we support supertopview (LabMice) and
            superquadruped (quadruped side-view animals).
        scale_list (list): A list of different resolutions for the spatial pyramid.
        video_extensions (string or Sequence[str], optional): When the input is a directory, only videos with
            these extensions are analyzed. Defaults to ``"mp4"``.
        adapt_iterations (int): Number of iterations for adaptation training.
            Empirically 1000 is sufficient. Training longer can cause worse
            performance depending whether there is occlusion in the video.
        modelfolder (string, optional): Because the API does not need a dlc project,
            the checkpoint and logs go to this temporary model folder, and otherwise
            model is saved to the current work place.
        customized_pose_config (string, optional): Path to a custom pose config for
            non-modelzoo models. Defaults to "".

    Examples:
        Create a SpatiotemporalAdaptation object and perform inference, adaptation, and post-adaptation inference:

            from deeplabcut.pose_estimation_tensorflow.modelzoo.api.spatiotemporal_adapt import (
                SpatiotemporalAdaptation,
            )
            video_path = "/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4"
            supermodel_name = "superanimal_topviewmouse"
            video_extensions = "mp4"
            adapter = SpatiotemporalAdaptation(
                video_path,
                supermodel_name,
                modelfolder="temp_topview",
                video_extensions=video_extensions,
            )
            adapter.before_adapt_inference()
            adapter.adaptation_training()
            adapter.after_adapt_inference()
    """
    if scale_list is None:
        scale_list = []

    self.video_path = video_path
    self.supermodel_name = supermodel_name
    self.scale_list = scale_list
    self.video_extensions = video_extensions
    vname = str(Path(self.video_path).stem)
    self.adapt_modelprefix = vname + "_video_adaptation"
    self.adapt_iterations = adapt_iterations
    self.modelfolder = modelfolder
    self.init_weights = init_weights

    project_name = "_".join(supermodel_name.split("_")[:-1])
    model_name = supermodel_name.split("_")[-1]
    self.project_name = project_name
    self.model_name = model_name

    if not customized_pose_config:
        dlc_root_path = get_deeplabcut_path()

        project_config = read_config(dlc_root_path / "modelzoo" / "project_configs" / f"{project_name}.yaml")

        model_config = read_config(dlc_root_path / "modelzoo" / "model_configs" / f"{model_name}.yaml")

        joints = [i for i in range(len(project_config["bodyparts"]))]
        num_joints = len(joints)
        model_config["all_joints"] = joints
        model_config["all_joints_names"] = project_config["bodyparts"]
        model_config["num_joints"] = num_joints
        model_config["num_limbs"] = int((num_joints * (num_joints - 1)) // 2)
        self.customized_pose_config = {**project_config, **model_config}
    else:
        self.customized_pose_config = customized_pose_config

adaptation_training 

adaptation_training(displayiters=500, saveiters=1000, **kwargs)

There should be two choices, either taking a config, with is then assuming there is a DLC project.

Or we make up a fake one, then we use a light way convention to do adaptation

Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs):
    """There should be two choices, either taking a config, with is then assuming
    there is a DLC project.

    Or we make up a fake one, then we use a light way convention to do adaptation
    """
    # looking for the pseudo label path
    DLCscorer = "DLC_" + Path(self.init_weights).stem
    vname = str(Path(self.video_path).stem)
    video_root = Path(self.video_path).parent

    _, pseudo_label_path, _, _ = load_analyzed_data(video_root, vname, DLCscorer, False, "")
    if self.modelfolder != "":
        Path(self.modelfolder).mkdir(parents=True, exist_ok=True)

    self.adapt_iterations = kwargs.get("adapt_iterations", self.adapt_iterations)

    self.train_without_project(
        pseudo_label_path,
        displayiters=displayiters,
        saveiters=saveiters,
        **kwargs,
    )