Feature/LABIMP 7887

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-merge-conflict
@@ -11,25 +11,25 @@ repos:
       - id: mixed-line-ending
 
   - repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 25.9.0
     hooks:
       - id: black
         args: ["--line-length=88"]
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
+    rev: 7.0.0
     hooks:
       - id: isort
         args: ["--profile=black", "--line-length=88"]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+    rev: 7.3.0
     hooks:
       - id: flake8
         additional_dependencies: []
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.18.2
     hooks:
       - id: mypy
         args: ["--config=pyproject.toml"]

labellerr/base/singleton.py

@@ -17,4 +17,4 @@ def __new__(cls, *args, **kwargs):
 
     def __init__(self, *args):
         if type(self) is Singleton:
-            raise TypeError("Can't instantiate Singleton class")
+            raise TypeError("Can't instantiate Singleton class")

labellerr/client.py

@@ -1040,7 +1040,7 @@ def upload_and_monitor():
                     extra_headers={"Origin": constants.ALLOWED_ORIGINS},
                 )
                 status_url = f"{self.base_url}/actions/upload_answers_status?project_id={self.project_id}&job_id={self.job_id}&client_id={self.client_id}"
-                
+
                 def check_job_status():
                     response = requests.request(
                         "GET", status_url, headers=headers, data={}
@@ -1066,7 +1066,7 @@ def on_exception(e):
                     on_success=on_success,
                     on_exception=on_exception
                 )
-                
+
                 return result
 
             except Exception as e:
@@ -1108,15 +1108,15 @@ def get_job_status():
                     "GET", url, headers=headers, data=payload
                 )
                 response_data = response.json()
-                
+
                 # Log current status for visibility
                 current_status = response_data.get('response', {}).get('status', 'unknown')
                 logging.info(f"Pre-annotation job status: {current_status}")
-                
+
                 # Check if job failed and raise error immediately
                 if current_status == 'failed':
                     raise LabellerrError('Internal server error: ', response_data)
-                
+
                 return response_data
 
             def is_job_completed(response_data):

labellerr/config.py

@@ -1,4 +1,3 @@
-"""This is to be removed, should be in constants.py
-"""
+"""This is to be removed, should be in constants.py"""
 
 cdn_server_address = "cdn-951134552678.us-central1.run.app:443"

labellerr/core/autolabel/__init__.py

@@ -1,2 +1 @@
-"""Inference core wrappers go here.
-"""
+"""Inference core wrappers go here."""

labellerr/core/datasets/__init__.py

@@ -1,7 +1,5 @@
-"""This module will contain all CRUD for datasets. Example, create, list datasets, get dataset, delete dataset, update dataset, etc.
-"""
-# from labellerr.core.datasets.base import LabellerrDataset
+"""This module will contain all CRUD for datasets. Example, create, list datasets, get dataset, delete dataset, update dataset, etc."""
 
-# __all__ = [
-#     'LabellerrDataset'
-#     ]
+from labellerr.core.datasets.base import LabellerrDataset
+
+__all__ = ["LabellerrDataset"]

labellerr/core/datasets/base.py

@@ -1,20 +1,19 @@
-from labellerr.client import LabellerrClient
-from labellerr.exceptions import LabellerrError
-from labellerr.core.files import LabellerrFile
-from labellerr import constants
 import uuid
-from abc import ABCMeta
-import pprint
+
+from labellerr import constants
+from labellerr.core.files import LabellerrFile
+from labellerr.exceptions import LabellerrError
+
 
 class LabellerrDataset:
     """
     Class for handling video dataset operations and fetching multiple video files.
     """
-    
-    def __init__(self, client: LabellerrClient, dataset_id: str, project_id: str):
+
+    def __init__(self, client, dataset_id: str, project_id: str):
         """
         Initialize video dataset instance.
-        
+
         :param client: LabellerrClient instance
         :param dataset_id: Dataset ID
         :param project_id: Project ID containing the dataset
@@ -23,139 +22,154 @@ def __init__(self, client: LabellerrClient, dataset_id: str, project_id: str):
         self.dataset_id = dataset_id
         self.project_id = project_id
         self.client_id = client.client_id
-    
+
     def fetch_files(self, page_size: int = 1000):
         """
         Fetch all video files in this dataset as LabellerrVideoFile instances.
-        
+
         :param page_size: Number of files to fetch per API request (default: 10)
         :return: List of file IDs
         """
         try:
             all_file_ids = []
             next_search_after = None  # Start with None for first page
-            
+
             while True:
                 unique_id = str(uuid.uuid4())
                 url = f"{constants.BASE_URL}/search/files/all"
                 params = {
-                    'sort_by': 'created_at',
-                    'sort_order': 'desc',
-                    'size': page_size,
-                    'uuid': unique_id,
-                    'dataset_id': self.dataset_id,
-                    'client_id': self.client_id
+                    "sort_by": "created_at",
+                    "sort_order": "desc",
+                    "size": page_size,
+                    "uuid": unique_id,
+                    "dataset_id": self.dataset_id,
+                    "client_id": self.client_id,
                 }
-                
+
                 # Add next_search_after only if it exists (don't send on first request)
                 if next_search_after:
-                    url+= f"?next_search_after={next_search_after}"
-                
+                    url += f"?next_search_after={next_search_after}"
+
                 # print(params)
-
-                response = self.client.make_api_request(self.client_id, url, params, unique_id)
-
+
+                response = self.client.make_api_request(
+                    self.client_id, url, params, unique_id
+                )
+
                 # pprint.pprint(response)
-                
+
                 # Extract files from the response
-                files = response.get('response', {}).get('files', [])
-                
+                files = response.get("response", {}).get("files", [])
+
                 # Collect file IDs
                 for file_info in files:
-                    file_id = file_info.get('file_id')
+                    file_id = file_info.get("file_id")
                     if file_id:
                         all_file_ids.append(file_id)
-                
+
                 # Get next_search_after for pagination
-                next_search_after = response.get('response', {}).get('next_search_after')
-
-
+                next_search_after = response.get("response", {}).get(
+                    "next_search_after"
+                )
+
                 # Break if no more pages or no files returned
                 if not next_search_after or not files:
                     break
-                
+
                 print(f"Fetched total: {len(all_file_ids)}")
-            
+
             print(f"Total file IDs extracted: {len(all_file_ids)}")
             # return all_file_ids
-            
+
             # Create LabellerrVideoFile instances for each file_id
             video_files = []
-            print(f"\nCreating LabellerrFile instances for {len(all_file_ids)} files...")
-
+            print(
+                f"\nCreating LabellerrFile instances for {len(all_file_ids)} files..."
+            )
+
             for file_id in all_file_ids:
                 try:
                     video_file = LabellerrFile(
                         client=self.client,
                         file_id=file_id,
                         project_id=self.project_id,
-                        dataset_id=self.dataset_id
+                        dataset_id=self.dataset_id,
                     )
                     video_files.append(video_file)
                 except Exception as e:
-                    print(f"Warning: Failed to create file instance for {file_id}: {str(e)}")
-
+                    print(
+                        f"Warning: Failed to create file instance for {file_id}: {str(e)}"
+                    )
+
             print(f"Successfully created {len(video_files)} LabellerrFile instances")
             return video_files
-        
+
         except Exception as e:
             raise LabellerrError(f"Failed to fetch dataset files: {str(e)}")
-    
+
     def download(self):
         """
-        Process all video files in the dataset: download frames, create videos, 
+        Process all video files in the dataset: download frames, create videos,
         and automatically clean up temporary files.
-        
+
         :param output_folder: Base folder where dataset folder will be created
         :return: List of processing results for all files
         """
         try:
             print(f"\n{'#'*70}")
             print(f"# Starting batch video processing for dataset: {self.dataset_id}")
             print(f"{'#'*70}\n")
-            
+
             # Fetch all video files
             video_files = self.fetch_files()
-            
+
             if not video_files:
                 print("No video files found in dataset")
                 return []
-            
+
             print(f"\nProcessing {len(video_files)} video files...\n")
-            
+
             results = []
             successful = 0
             failed = 0
-            
+
             print(f"\nStarting download of {len(video_files)} files...")
             for idx, video_file in enumerate(video_files, 1):
                 try:
                     # Call the new all-in-one method
                     result = video_file.download_create_video_auto_cleanup()
                     results.append(result)
                     successful += 1
-                    print(f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)", end="", flush=True)
-
+                    print(
+                        f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)",
+                        end="",
+                        flush=True,
+                    )
+
                 except Exception as e:
                     error_result = {
-                        'status': 'failed',
-                        'file_id': video_file.file_id,
-                        'error': str(e)
+                        "status": "failed",
+                        "file_id": video_file.file_id,
+                        "error": str(e),
                     }
                     results.append(error_result)
                     failed += 1
-                    print(f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)", end="", flush=True)
-
+                    print(
+                        f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)",
+                        end="",
+                        flush=True,
+                    )
+
             # Summary
             print(f"\n{'#'*70}")
-            print(f"# Batch Processing Complete")
+            print("# Batch Processing Complete")
             print(f"# Total files: {len(video_files)}")
             print(f"# Successful: {successful}")
             print(f"# Failed: {failed}")
             print(f"{'#'*70}\n")
-            
+
             return results
-                
+
         except Exception as e:
             raise LabellerrError(f"Failed to process dataset videos: {str(e)}")
 
@@ -165,16 +179,16 @@ def download(self):
 #     api_key = ""
 #     api_secret = ""
 #     client_id = ""
-    
+
 #     dataset_id = "59438ec3-12e0-4687-8847-1e6e01b0bf25"
 #     project_id = "farrah_supposed_hookworm_34155"
-    
+
 #     client = LabellerrClient(api_key, api_secret, client_id)
-    
+
 #     dataset = LabellerrVideoDataset(client, dataset_id, project_id)
-    
+
 #     # Process all videos in the dataset
 #     results = dataset.download()
-    
+
 #     # Print summary
-#     pprint.pprint(results)
+#     pprint.pprint(results)

labellerr/core/files/__init__.py

@@ -7,8 +7,8 @@
 from labellerr.core.files.video_file import LabellerrVideoFile
 
 __all__ = [
-    'LabellerrFile',
-    'LabellerrImageFile', 
-    'LabellerrVideoFile',
-    'LabellerrFileMeta'
-]
+    "LabellerrFile",
+    "LabellerrImageFile",
+    "LabellerrVideoFile",
+    "LabellerrFileMeta",
+]