Skip to content

deeplabcut.pose_tracking_pytorch.apis

Functions:

Name Description
transformer_reID

Enables tracking with transformer.

transformer_reID 

transformer_reID(
    config: str,
    videos: list[str],
    video_extensions: str | Sequence[str] | None = None,
    shuffle: int = 1,
    trainingsetindex: int = 0,
    track_method: str = "ellipse",
    n_tracks: int | None = None,
    n_triplets: int = 1000,
    train_epochs: int = 100,
    train_frac: float = 0.8,
    modelprefix: str = "",
    destfolder: str = None,
)

Enables tracking with transformer.

Substeps include
  • Mines triplets from tracklets in videos (from another tracker)
  • These triplets are later used to tran a transformer with triplet loss
  • The transformer derived appearance similarity is then used as a stitching loss when tracklets are stitched during tracking.

Outputs: The tracklet file is saved in the same folder where the non-transformer tracklet file is stored.

Parameters

config: string Full path of the config.yaml file as a string.

list

A list of strings containing the full paths to videos for analysis or a path to the directory, where all the videos with same extension are stored.

str | Sequence[str] | None, optional, default=None

Controls how videos are filtered, based on file extension. File paths and directory contents are treated differently: - None (default): file paths are accepted as-is; directories are scanned for files with a recognized video extension. - str or Sequence[str] (e.g. "mp4" or ["mp4", "avi"]): both file paths and directory contents are filtered by the given extension(s).

int, optional

which shuffle to use

int. optional

which training fraction to use, identified by its index

str, optional

track method from which tracklets are sampled

int

number of tracks to be formed in the videos. TODO: handling videos with different number of tracks

(optional) int

number of triplets to be mined from the videos

(optional), int

number of epochs to train the transformer

(optional), fraction

fraction of triplets used for training/testing of the transformer

Examples

Training model for one video based on ellipse-tracker derived tracklets

config = "/home/users/.../dlc-project-2025-01-01/config.yaml" videos = ['/home/alex/video.mp4'] deeplabcut.transformer_reID(config, videos, shuffle=1, track_method="ellipse") deeplabcut.create_labeled_video( config, videos, shuffle=1, track_method="transformer", )

Source code in deeplabcut/pose_tracking_pytorch/apis.py 
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
@renamed_parameter(old="videotype", new="video_extensions", since="3.0.0")
def transformer_reID(
    config: str,
    videos: list[str],
    video_extensions: str | Sequence[str] | None = None,
    shuffle: int = 1,
    trainingsetindex: int = 0,
    track_method: str = "ellipse",
    n_tracks: int | None = None,
    n_triplets: int = 1000,
    train_epochs: int = 100,
    train_frac: float = 0.8,
    modelprefix: str = "",
    destfolder: str = None,
):
    """Enables tracking with transformer.

    Substeps include:
        - Mines triplets from tracklets in videos (from another tracker)
        - These triplets are later used to tran a transformer with triplet loss
        - The transformer derived appearance similarity is then used as a stitching loss
            when tracklets are stitched during tracking.

    Outputs: The tracklet file is saved in the same folder where the non-transformer
    tracklet file is stored.

    Parameters
    ----------
    config: string
        Full path of the config.yaml file as a string.

    videos: list
        A list of strings containing the full paths to videos for analysis or a path to
        the directory, where all the videos with same extension are stored.

    video_extensions : str | Sequence[str] | None, optional, default=None
        Controls how ``videos`` are filtered, based on file extension.
        File paths and directory contents are treated differently:
        - ``None`` (default): file paths are accepted as-is; directories are
          scanned for files with a recognized video extension.
        - ``str`` or ``Sequence[str]`` (e.g. ``"mp4"`` or ``["mp4", "avi"]``):
          both file paths and directory contents are filtered by the given
          extension(s).

    shuffle : int, optional
        which shuffle to use

    trainingsetindex : int. optional
        which training fraction to use, identified by its index

    track_method: str, optional
        track method from which tracklets are sampled

    n_tracks: int
        number of tracks to be formed in the videos.
        TODO: handling videos with different number of tracks

    n_triplets: (optional) int
        number of triplets to be mined from the videos

    train_epochs: (optional), int
        number of epochs to train the transformer

    train_frac: (optional), fraction
        fraction of triplets used for training/testing of the transformer

    Examples
    --------

    Training model for one video based on ellipse-tracker derived tracklets
    >>> config = "/home/users/.../dlc-project-2025-01-01/config.yaml"
    >>> videos = ['/home/alex/video.mp4']
    >>> deeplabcut.transformer_reID(config, videos, shuffle=1, track_method="ellipse")
    >>> deeplabcut.create_labeled_video(
    >>>     config,
    >>>     videos,
    >>>     shuffle=1,
    >>>     track_method="transformer",
    >>> )
    --------
    """
    import os

    import deeplabcut
    from deeplabcut.utils import auxiliaryfunctions

    # calling create_tracking_dataset, train_tracking_transformer, stitch_tracklets

    cfg = auxiliaryfunctions.read_config(config)

    DLCscorer, _ = deeplabcut.utils.auxiliaryfunctions.GetScorerName(
        cfg,
        shuffle=shuffle,
        trainFraction=cfg["TrainingFraction"][trainingsetindex],
        modelprefix=modelprefix,
    )

    deeplabcut.compat.create_tracking_dataset(
        config,
        videos,
        track_method,
        video_extensions=video_extensions,
        shuffle=shuffle,
        trainingsetindex=trainingsetindex,
        modelprefix=modelprefix,
        n_triplets=n_triplets,
        destfolder=destfolder,
    )

    (
        trainposeconfigfile,
        testposeconfigfile,
        snapshotfolder,
    ) = deeplabcut.return_train_network_path(
        config,
        shuffle=shuffle,
        modelprefix=modelprefix,
        trainingsetindex=trainingsetindex,
    )

    deeplabcut.pose_tracking_pytorch.train_tracking_transformer(
        config,
        DLCscorer,
        videos,
        video_extensions=video_extensions,
        train_frac=train_frac,
        modelprefix=modelprefix,
        train_epochs=train_epochs,
        ckpt_folder=snapshotfolder,
        destfolder=destfolder,
    )

    transformer_checkpoint = os.path.join(snapshotfolder, f"dlc_transreid_{train_epochs}.pth")

    if not os.path.exists(transformer_checkpoint):
        raise FileNotFoundError(f"checkpoint {transformer_checkpoint} not found")

    deeplabcut.stitch_tracklets(
        config,
        videos,
        video_extensions=video_extensions,
        shuffle=shuffle,
        trainingsetindex=trainingsetindex,
        track_method=track_method,
        modelprefix=modelprefix,
        n_tracks=n_tracks,
        transformer_checkpoint=transformer_checkpoint,
        destfolder=destfolder,
    )