diff --git a/plugins/beetslabels.py b/plugins/beetslabels.py
index ce11a48..f6a7b34 100644
--- a/plugins/beetslabels.py
+++ b/plugins/beetslabels.py
@@ -14,16 +14,20 @@ def do_modify_labels(lib, objs, label, action):
     changed = []
 
     for obj in objs:
-        labels = []
+        labels = {}
         if action != 2:
             if "mylabels" in obj:
                 labels = json.loads(obj["mylabels"])
 
+            split = label.split(":")
+
             if action == 0:
-                labels.append(label)
-                labels = sorted(set(labels))
+                if len(split) == 1:
+                    labels[label] = 0
+                else:
+                    labels[split[0]] = int(split[1])
             elif label in labels:
-                labels.remove(label)
+                del labels[label]
 
         obj_mods = {
             "mylabels": json.dumps(labels)