operations module

datumaro.components.operations.match_annotations_equal(a, b)[source]
datumaro.components.operations.merge_annotations_equal(a, b)[source]
datumaro.components.operations.merge_categories(sources)[source]
class datumaro.components.operations.MergingStrategy(**options)[source]

Bases: datumaro.components.cli_plugin.CliPlugin

classmethod merge(sources, **options)[source]
__init__(**options)[source]
class datumaro.components.operations.ExactMerge[source]

Bases: object

Merges several datasets using the “simple” algorithm: - items are matched by (id, subset) pairs - matching items share the media info available:

  • nothing + nothing = nothing

  • nothing + something = something

  • something A + something B = conflict

  • annotations are matched by value and shared

  • in case of conflicts, throws an error

classmethod merge(*sources: datumaro.components.extractor.IExtractor) → datumaro.components.dataset.DatasetItemStorage[source]
static merge_categories(sources: Iterable[datumaro.components.extractor.IExtractor]) → Dict[datumaro.components.annotation.AnnotationType, datumaro.components.annotation.Categories][source]
classmethod merge(*sources: datumaro.components.extractor.IExtractor) → datumaro.components.dataset.DatasetItemStorage[source]
static merge_categories(sources: Iterable[datumaro.components.extractor.IExtractor]) → Dict[datumaro.components.annotation.AnnotationType, datumaro.components.annotation.Categories][source]
class datumaro.components.operations.IntersectMerge(conf=_Nothing.NOTHING)[source]

Bases: datumaro.components.operations.MergingStrategy

Error trackers:

attrib
add_item_error(error, *args, **kwargs)[source]

Indexes:

_dataset_map

id(dataset) -> (dataset, index)

_item_map

id(item) -> (item, id(dataset))

_ann_map

id(ann) -> (ann, id(item))

_item_id
_item

Misc.

_categories = attrib(init=False)  merged categories
class Conf(*, pairwise_dist=0.5, sigma=_Nothing.NOTHING, output_conf_thresh=0, quorum=0, ignored_attributes=_Nothing.NOTHING, groups=_Nothing.NOTHING, close_distance=0.75)[source]

Bases: object

__eq__(other)

Method generated by attrs for class IntersectMerge.Conf.

__init__(*, pairwise_dist=0.5, sigma=_Nothing.NOTHING, output_conf_thresh=0, quorum=0, ignored_attributes=_Nothing.NOTHING, groups=_Nothing.NOTHING, close_distance=0.75) → None

Method generated by attrs for class IntersectMerge.Conf.

add_item_error(error, *args, **kwargs)[source]
get_ann_source(ann_id)[source]
merge_items(items)[source]
merge_annotations(sources)[source]
static match_items(datasets)[source]
__eq__(other)

Method generated by attrs for class IntersectMerge.

__init__(conf=_Nothing.NOTHING) → None

Method generated by attrs for class IntersectMerge.

class datumaro.components.operations.AnnotationMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None)[source]
class datumaro.components.operations.LabelMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None)[source]
class datumaro.components.operations._ShapeMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations.AnnotationMatcher

pairwise_dist = attrib(converter=float, default=0.9)
cluster_dist = attrib(converter=float, default=-1.0)
match_annotations(sources)[source]
static distance(a, b)[source]
label_matcher(a, b)[source]
__eq__(other)

Method generated by attrs for class _ShapeMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class _ShapeMatcher.

class datumaro.components.operations.BboxMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations._ShapeMatcher

__eq__(other)

Method generated by attrs for class BboxMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class BboxMatcher.

class datumaro.components.operations.PolygonMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations._ShapeMatcher

__eq__(other)

Method generated by attrs for class PolygonMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class PolygonMatcher.

class datumaro.components.operations.MaskMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations._ShapeMatcher

__eq__(other)

Method generated by attrs for class MaskMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class MaskMatcher.

class datumaro.components.operations.PointsMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, sigma: Optional[list] = None, instance_map)[source]

Bases: datumaro.components.operations._ShapeMatcher

sigma: Optional[list] = attrib(default=None)
instance_map = attrib(converter=dict)
sigma: Optional[list]
distance(a, b)[source]
__eq__(other)

Method generated by attrs for class PointsMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, sigma: Optional[list] = None, instance_map) → None

Method generated by attrs for class PointsMatcher.

class datumaro.components.operations.LineMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations._ShapeMatcher

static distance(a, b)[source]
__eq__(other)

Method generated by attrs for class LineMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class LineMatcher.

class datumaro.components.operations.CaptionsMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None)[source]

Bases: datumaro.components.operations.AnnotationMatcher

match_annotations(sources)[source]
__eq__(other)

Method generated by attrs for class CaptionsMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None) → None

Method generated by attrs for class CaptionsMatcher.

class datumaro.components.operations.Cuboid3dMatcher(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0)[source]

Bases: datumaro.components.operations._ShapeMatcher

static distance(a, b)[source]
__eq__(other)

Method generated by attrs for class Cuboid3dMatcher.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0) → None

Method generated by attrs for class Cuboid3dMatcher.

class datumaro.components.operations.AnnotationMerger[source]

Bases: object

merge_clusters(clusters)[source]
__eq__(other)

Method generated by attrs for class AnnotationMerger.

__init__() → None

Method generated by attrs for class AnnotationMerger.

class datumaro.components.operations.LabelMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, quorum=0)[source]

Bases: datumaro.components.operations.AnnotationMerger, datumaro.components.operations.LabelMatcher

quorum = attrib(converter=int, default=0)
merge_clusters(clusters)[source]
__eq__(other)

Method generated by attrs for class LabelMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, quorum=0) → None

Method generated by attrs for class LabelMerger.

class datumaro.components.operations._ShapeMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations.AnnotationMerger, datumaro.components.operations._ShapeMatcher

quorum = attrib(converter=int, default=0)
merge_clusters(clusters)[source]
find_cluster_label(cluster)[source]
merge_cluster_shape(cluster)[source]
merge_cluster(cluster)[source]
__eq__(other)

Method generated by attrs for class _ShapeMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class _ShapeMerger.

class datumaro.components.operations.BboxMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.BboxMatcher

__eq__(other)

Method generated by attrs for class BboxMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class BboxMerger.

class datumaro.components.operations.PolygonMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.PolygonMatcher

__eq__(other)

Method generated by attrs for class PolygonMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class PolygonMerger.

class datumaro.components.operations.MaskMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.MaskMatcher

__eq__(other)

Method generated by attrs for class MaskMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class MaskMerger.

class datumaro.components.operations.PointsMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0, sigma: Optional[list] = None, instance_map)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.PointsMatcher

__eq__(other)

Method generated by attrs for class PointsMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0, sigma: Optional[list] = None, instance_map) → None

Method generated by attrs for class PointsMerger.

sigma: Optional[list]
class datumaro.components.operations.LineMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.LineMatcher

__eq__(other)

Method generated by attrs for class LineMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class LineMerger.

class datumaro.components.operations.CaptionsMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None)[source]

Bases: datumaro.components.operations.AnnotationMerger, datumaro.components.operations.CaptionsMatcher

__eq__(other)

Method generated by attrs for class CaptionsMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None) → None

Method generated by attrs for class CaptionsMerger.

class datumaro.components.operations.Cuboid3dMerger(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0)[source]

Bases: datumaro.components.operations._ShapeMerger, datumaro.components.operations.Cuboid3dMatcher

merge_cluster(cluster)[source]
__eq__(other)

Method generated by attrs for class Cuboid3dMerger.

__init__(*, context: Optional[datumaro.components.operations.IntersectMerge] = None, pairwise_dist=0.9, cluster_dist=- 1.0, quorum=0) → None

Method generated by attrs for class Cuboid3dMerger.

datumaro.components.operations.match_segments(a_segms, b_segms, distance=<function segment_iou>, dist_thresh=1.0, label_matcher=<function <lambda>>)[source]
a_matches = -np.ones(len(a_segms), dtype=int)
b_matches = -np.ones(len(b_segms), dtype=int)

indices of b_segms matched to a bboxes indices of a_segms matched to b bboxes

matches = []
mispred = []

matches: boxes we succeeded to match completely mispred: boxes we succeeded to match, having label mismatch

class datumaro.components.operations.mean_std(dataset: datumaro.components.extractor.IExtractor)[source]

Bases:

datumaro.components.operations.compute_image_statistics(dataset: datumaro.components.extractor.IExtractor)[source]
datumaro.components.operations.compute_ann_statistics(dataset: datumaro.components.extractor.IExtractor)[source]
datumaro.components.operations.get_label()
datumaro.components.operations.total_pixels()

numpy.sum might be faster, but could overflow with large datasets. Python’s int can transparently mutate to be of indefinite precision (long)

class datumaro.components.operations.DistanceComparator(iou_threshold=0.5)[source]

Bases: object

iou_threshold = attrib(converter=float, default=0.5)
match_annotations(item_a, item_b)[source]
match_labels(item_a, item_b)[source]
match_polygons(item_a, item_b)[source]
match_masks(item_a, item_b)[source]
match_boxes(item_a, item_b)[source]
match_points(item_a, item_b)[source]
match_lines(item_a, item_b)[source]
__eq__(other)

Method generated by attrs for class DistanceComparator.

__init__(iou_threshold=0.5) → None

Method generated by attrs for class DistanceComparator.

datumaro.components.operations.match_items_by_id(a: datumaro.components.extractor.IExtractor, b: datumaro.components.extractor.IExtractor)[source]
datumaro.components.operations.match_items_by_image_hash(a: datumaro.components.extractor.IExtractor, b: datumaro.components.extractor.IExtractor)[source]
datumaro.components.operations.find_unique_images(dataset: datumaro.components.extractor.IExtractor, item_hash: Optional[Callable] = None)[source]
datumaro.components.operations.match_classes(a: Dict[datumaro.components.annotation.AnnotationType, datumaro.components.annotation.Categories], b: Dict[datumaro.components.annotation.AnnotationType, datumaro.components.annotation.Categories])[source]
class datumaro.components.operations.ExactComparator(*, match_images: bool = False, ignored_fields=_Nothing.NOTHING, ignored_attrs=_Nothing.NOTHING, ignored_item_attrs=_Nothing.NOTHING)[source]

Bases: object

match_images: bool = attrib(kw_only=True, default=False)
ignored_fields = attrib(kw_only=True,
factory=set, validator=default_if_none(set))
ignored_attrs = attrib(kw_only=True,
factory=set, validator=default_if_none(set))
ignored_item_attrs = attrib(kw_only=True,
factory=set, validator=default_if_none(set))
match_images: bool
errors: list
compare_datasets(a, b)[source]
__eq__(other)

Method generated by attrs for class ExactComparator.

__init__(*, match_images: bool = False, ignored_fields=_Nothing.NOTHING, ignored_attrs=_Nothing.NOTHING, ignored_item_attrs=_Nothing.NOTHING) → None

Method generated by attrs for class ExactComparator.