push image fixes and builder example(not done)

basedformer/dataset.py

@@ -149,17 +149,23 @@ class ImageDatasetBuilder():
 
     @property
     def biggest_id(self):
-        if self.index is None:
-            return -1
-        else:
+        try:
             return np.max(self.np_index[:, 2])
+        except:
+            return -1
             
     @property
     def biggest_item(self):
-        if self.index is None:
-            return -1
-        else:
+        try:
             return np.max(self.np_index[:, 1])
+        except:
+            return -1
+    @property
+    def total_ids(self):
+        try:
+            return len(self.np_index)
+        except:
+            return -1
 
     @property
     def np_index(self):
@@ -208,9 +214,9 @@ class ImageDatasetBuilder():
         else:
             raise Exception("Metadata file not found at {}".format(self.metadata_path))
 
-    def operate(self, operation, data_batch, identities, metadata=None):
+    def operate(self, operation, batch, identities, metadata=None):
         executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.threads)
-        futures = executor.map(operation, data_batch)
+        futures = executor.map(operation, batch)
         futures = list(futures)
             
         for data, identity in zip(futures, identities):

scripts/builder.py

+from basedformer import dataset
+import simplejpeg
+builder = dataset.ImageDatasetBuilder("/home/xuser/nvme1/dataset/buildtest", "e621", metadata=True)
+builder.build()
+
+def encode_op(self, file_path):
+    f = open(file_path, "rb")
+    data = f.read()
+    f.close()
+
+    if simplejpeg.is_jpeg(data):
+        pass
+    else:
+        data = simplejpeg.encode_jpeg(data, quality=91)
+    return data
+
+builder.operate(encode_op, batch, danbooru_ids)
+
+