Skip to content

deeplabcut.pose_estimation_tensorflow.modelzoo.api.spatiotemporal_adapt

Classes:

Name Description
SpatiotemporalAdaptation

SpatiotemporalAdaptation

Methods:

Name Description
__init__

This class supports video adaptation to a super model.
adaptation_training

There should be two choices, either taking a config, with is then assuming
Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
class SpatiotemporalAdaptation:
    @renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
    def __init__(
        self,
        video_path,
        supermodel_name,
        scale_list=None,
        video_extensions: str | Sequence[str] | None = "mp4",
        adapt_iterations=1000,
        modelfolder="",
        customized_pose_config="",
        init_weights="",
    ):
        """This class supports video adaptation to a super model.

        Parameters
        ----------
        video_path: string
           The string to the path of the video
        init_weights: string
           The path to a superanimal model's checkpoint
        supermodel_name: string
           Currently we support supertopview(LabMice) and superquadruped (quadruped side-view animals)
        scale_list: list
           A list of different resolutions for the spatial pyramid
        video_extensions: str | Sequence[str] | None, default=None
        Controls how ``videos`` are filtered, based on file extension.
        File paths and directory contents are treated differently:
        - ``None`` (default): file paths are accepted as-is; directories are
          scanned for files with a recognized video extension.
        - ``str`` or ``Sequence[str]`` (e.g. ``"mp4"`` or ``["mp4", "avi"]``):
          both file paths and directory contents are filtered by the given
          extension(s).
        adapt_iterations: int
           Number of iterations for adaptation training. Empirically 1000 is sufficient. Training longer can cause worse
           performance depending whether there is occlusion in the video
        modelfolder: string, optional
           Because the API does not need a dlc project, the checkpoint and logs go to this temporary model folder, and
           otherwise model is saved to the current work place
        customized_pose_config: string, optional
           For future support of non modelzoo model

        Examples
        --------

        from  deeplabcut.modelzoo.apis import SpatiotemporalAdaptation
        video_path = '/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4'
        superanimal_name = 'superanimal_topviewmouse'
        video_extensions = 'mp4'
        >>> adapter = SpatiotemporalAdaptation(video_path,
                                       superanimal_name,
                                       modelfolder = "temp_topview",
                                       video_extensions = video_extensions)

        adapter.before_adapt_inference()
        adapter.adaptation_training()
        adapter.after_adapt_inference()
        """
        if scale_list is None:
            scale_list = []

        self.video_path = video_path
        self.supermodel_name = supermodel_name
        self.scale_list = scale_list
        self.video_extensions = video_extensions
        vname = str(Path(self.video_path).stem)
        self.adapt_modelprefix = vname + "_video_adaptation"
        self.adapt_iterations = adapt_iterations
        self.modelfolder = modelfolder
        self.init_weights = init_weights

        project_name = "_".join(supermodel_name.split("_")[:-1])
        model_name = supermodel_name.split("_")[-1]
        self.project_name = project_name
        self.model_name = model_name

        if not customized_pose_config:
            dlc_root_path = get_deeplabcut_path()

            project_config = read_config(
                os.path.join(dlc_root_path, "modelzoo", "project_configs", f"{project_name}.yaml")
            )

            model_config = read_config(os.path.join(dlc_root_path, "modelzoo", "model_configs", f"{model_name}.yaml"))

            joints = [i for i in range(len(project_config["bodyparts"]))]
            num_joints = len(joints)
            model_config["all_joints"] = joints
            model_config["all_joints_names"] = project_config["bodyparts"]
            model_config["num_joints"] = num_joints
            model_config["num_limbs"] = int((num_joints * (num_joints - 1)) // 2)
            self.customized_pose_config = {**project_config, **model_config}
        else:
            self.customized_pose_config = customized_pose_config

    def before_adapt_inference(self, make_video=False, **kwargs):
        if self.init_weights != "":
            print("using customized weights", self.init_weights)
            _, datafiles = video_inference(
                [self.video_path],
                self.project_name,
                self.model_name,
                video_extensions=self.video_extensions,
                scale_list=self.scale_list,
                init_weights=self.init_weights,
                customized_test_config=self.customized_pose_config,
            )
        else:
            self.init_weights, datafiles = video_inference(
                [self.video_path],
                self.project_name,
                self.model_name,
                video_extensions=self.video_extensions,
                scale_list=self.scale_list,
                customized_test_config=self.customized_pose_config,
            )
        if kwargs.pop("plot_trajectories", True):
            if len(datafiles) == 0:
                print("No data files found for plotting trajectory")
            else:
                _plot_trajectories(datafiles[0])

        if make_video:
            create_labeled_video(
                "",
                [self.video_path],
                video_extensions=self.video_extensions,
                filtered=False,
                init_weights=self.init_weights,
                draw_skeleton=True,
                superanimal_name=self.project_name,
                **kwargs,
            )

    def train_without_project(self, pseudo_label_path, **kwargs):
        from deeplabcut.pose_estimation_tensorflow.core.train_multianimal import train

        displayiters = kwargs.pop("displayiters", 500)
        saveiters = kwargs.pop("saveiters", 1000)
        self.adapt_iterations = kwargs.pop("adapt_iterations", self.adapt_iterations)

        train(
            self.customized_pose_config,
            displayiters=displayiters,
            saveiters=saveiters,
            maxiters=self.adapt_iterations,
            modelfolder=self.modelfolder,
            init_weights=self.init_weights,
            pseudo_labels=pseudo_label_path,
            video_path=self.video_path,
            superanimal=self.supermodel_name,
            **kwargs,
        )

    def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs):
        """There should be two choices, either taking a config, with is then assuming
        there is a DLC project.

        Or we make up a fake one, then we use a light way convention to do adaptation
        """

        # looking for the pseudo label path
        DLCscorer = "DLC_" + Path(self.init_weights).stem
        vname = str(Path(self.video_path).stem)
        video_root = Path(self.video_path).parent

        _, pseudo_label_path, _, _ = load_analyzed_data(video_root, vname, DLCscorer, False, "")
        if self.modelfolder != "":
            os.makedirs(self.modelfolder, exist_ok=True)

        self.adapt_iterations = kwargs.get("adapt_iterations", self.adapt_iterations)

        self.train_without_project(
            pseudo_label_path,
            displayiters=displayiters,
            saveiters=saveiters,
            **kwargs,
        )

    def after_adapt_inference(self, create_labeled_video, **kwargs):
        pattern = os.path.join(self.modelfolder, f"snapshot-{self.adapt_iterations}.index")
        ref_proj_config_path = ""

        files = glob.glob(pattern)

        if not len(files):
            raise ValueError("Weights were not found.")

        adapt_weights = files[0].replace(".index", "")

        # spatial pyramid is not for adapted model

        scale_list = kwargs.pop("scale_list", [])

        # spatial pyramid can still be useful for reducing jittering and quantization error

        _, datafiles = video_inference(
            [self.video_path],
            self.project_name,
            self.model_name,
            video_extensions=self.video_extensions,
            init_weights=adapt_weights,
            scale_list=scale_list,
            customized_test_config=self.customized_pose_config,
        )

        if kwargs.pop("plot_trajectories", True):
            _plot_trajectories(datafiles[0])

        if create_labeled_video:
            create_labeled_video(
                ref_proj_config_path,
                [self.video_path],
                video_extensions=self.video_extensions,
                filtered=False,
                init_weights=adapt_weights,
                draw_skeleton=True,
                superanimal_name=self.project_name,
                **kwargs,
            )

__init__ 

__init__(
    video_path,
    supermodel_name,
    scale_list=None,
    video_extensions: str | Sequence[str] | None = "mp4",
    adapt_iterations=1000,
    modelfolder="",
    customized_pose_config="",
    init_weights="",
)

This class supports video adaptation to a super model.

Parameters

video_path: string The string to the path of the video init_weights: string The path to a superanimal model's checkpoint supermodel_name: string Currently we support supertopview(LabMice) and superquadruped (quadruped side-view animals) scale_list: list A list of different resolutions for the spatial pyramid video_extensions: str | Sequence[str] | None, default=None Controls how videos are filtered, based on file extension. File paths and directory contents are treated differently: - None (default): file paths are accepted as-is; directories are scanned for files with a recognized video extension. - str or Sequence[str] (e.g. "mp4" or ["mp4", "avi"]): both file paths and directory contents are filtered by the given extension(s). adapt_iterations: int Number of iterations for adaptation training. Empirically 1000 is sufficient. Training longer can cause worse performance depending whether there is occlusion in the video modelfolder: string, optional Because the API does not need a dlc project, the checkpoint and logs go to this temporary model folder, and otherwise model is saved to the current work place customized_pose_config: string, optional For future support of non modelzoo model

Examples

from deeplabcut.modelzoo.apis import SpatiotemporalAdaptation video_path = '/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4' superanimal_name = 'superanimal_topviewmouse' video_extensions = 'mp4'

adapter = SpatiotemporalAdaptation(video_path, superanimal_name, modelfolder = "temp_topview", video_extensions = video_extensions)

adapter.before_adapt_inference() adapter.adaptation_training() adapter.after_adapt_inference()

Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
@renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
def __init__(
    self,
    video_path,
    supermodel_name,
    scale_list=None,
    video_extensions: str | Sequence[str] | None = "mp4",
    adapt_iterations=1000,
    modelfolder="",
    customized_pose_config="",
    init_weights="",
):
    """This class supports video adaptation to a super model.

    Parameters
    ----------
    video_path: string
       The string to the path of the video
    init_weights: string
       The path to a superanimal model's checkpoint
    supermodel_name: string
       Currently we support supertopview(LabMice) and superquadruped (quadruped side-view animals)
    scale_list: list
       A list of different resolutions for the spatial pyramid
    video_extensions: str | Sequence[str] | None, default=None
    Controls how ``videos`` are filtered, based on file extension.
    File paths and directory contents are treated differently:
    - ``None`` (default): file paths are accepted as-is; directories are
      scanned for files with a recognized video extension.
    - ``str`` or ``Sequence[str]`` (e.g. ``"mp4"`` or ``["mp4", "avi"]``):
      both file paths and directory contents are filtered by the given
      extension(s).
    adapt_iterations: int
       Number of iterations for adaptation training. Empirically 1000 is sufficient. Training longer can cause worse
       performance depending whether there is occlusion in the video
    modelfolder: string, optional
       Because the API does not need a dlc project, the checkpoint and logs go to this temporary model folder, and
       otherwise model is saved to the current work place
    customized_pose_config: string, optional
       For future support of non modelzoo model

    Examples
    --------

    from  deeplabcut.modelzoo.apis import SpatiotemporalAdaptation
    video_path = '/mnt/md0/shaokai/openfield_video/m3v1mp4.mp4'
    superanimal_name = 'superanimal_topviewmouse'
    video_extensions = 'mp4'
    >>> adapter = SpatiotemporalAdaptation(video_path,
                                   superanimal_name,
                                   modelfolder = "temp_topview",
                                   video_extensions = video_extensions)

    adapter.before_adapt_inference()
    adapter.adaptation_training()
    adapter.after_adapt_inference()
    """
    if scale_list is None:
        scale_list = []

    self.video_path = video_path
    self.supermodel_name = supermodel_name
    self.scale_list = scale_list
    self.video_extensions = video_extensions
    vname = str(Path(self.video_path).stem)
    self.adapt_modelprefix = vname + "_video_adaptation"
    self.adapt_iterations = adapt_iterations
    self.modelfolder = modelfolder
    self.init_weights = init_weights

    project_name = "_".join(supermodel_name.split("_")[:-1])
    model_name = supermodel_name.split("_")[-1]
    self.project_name = project_name
    self.model_name = model_name

    if not customized_pose_config:
        dlc_root_path = get_deeplabcut_path()

        project_config = read_config(
            os.path.join(dlc_root_path, "modelzoo", "project_configs", f"{project_name}.yaml")
        )

        model_config = read_config(os.path.join(dlc_root_path, "modelzoo", "model_configs", f"{model_name}.yaml"))

        joints = [i for i in range(len(project_config["bodyparts"]))]
        num_joints = len(joints)
        model_config["all_joints"] = joints
        model_config["all_joints_names"] = project_config["bodyparts"]
        model_config["num_joints"] = num_joints
        model_config["num_limbs"] = int((num_joints * (num_joints - 1)) // 2)
        self.customized_pose_config = {**project_config, **model_config}
    else:
        self.customized_pose_config = customized_pose_config

adaptation_training 

adaptation_training(displayiters=500, saveiters=1000, **kwargs)

There should be two choices, either taking a config, with is then assuming there is a DLC project.

Or we make up a fake one, then we use a light way convention to do adaptation

Source code in deeplabcut/pose_estimation_tensorflow/modelzoo/api/spatiotemporal_adapt.py 
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def adaptation_training(self, displayiters=500, saveiters=1000, **kwargs):
    """There should be two choices, either taking a config, with is then assuming
    there is a DLC project.

    Or we make up a fake one, then we use a light way convention to do adaptation
    """

    # looking for the pseudo label path
    DLCscorer = "DLC_" + Path(self.init_weights).stem
    vname = str(Path(self.video_path).stem)
    video_root = Path(self.video_path).parent

    _, pseudo_label_path, _, _ = load_analyzed_data(video_root, vname, DLCscorer, False, "")
    if self.modelfolder != "":
        os.makedirs(self.modelfolder, exist_ok=True)

    self.adapt_iterations = kwargs.get("adapt_iterations", self.adapt_iterations)

    self.train_without_project(
        pseudo_label_path,
        displayiters=displayiters,
        saveiters=saveiters,
        **kwargs,
    )