From e40915594c703b09e0a1c94b86f554327c802e90 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:03:36 +0530 Subject: [PATCH 01/10] [LABIMP-8954] fixed export for files more than 5000 --- .gitignore | 1 + labellerr/core/projects/base.py | 126 ++++++++++++++++++++++---------- labellerr/core/schemas.py | 2 +- labellerr/core/schemas/files.py | 2 +- 4 files changed, 92 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index a697ecb..9db31ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Python .venv +venv __pycache__ *.pyc */*.pyc diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index 594fdb5..f0cb85b 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -441,6 +441,93 @@ def upload_preannotations( else: return future.result() + def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): + # 1. Fetch all files from project to get slice_id and file_ids + file_ids = [] + slice_id = None + next_search_after = None + first_page = True + + while True: + files_res = self.list_files(search_queries=[], size=1000, next_search_after=next_search_after) + response_data = files_res.get("response", {}) + + if first_page: + slice_id = response_data.get("slice_id") + first_page = False + + files_list = response_data.get("files", []) + if not files_list: + break + + for file_item in files_list: + file_status = file_item.get("status") + # Filter by status if statuses are provided + if not export_config.statuses or file_status in export_config.statuses: + file_ids.append(file_item.get("file_id")) + + next_search_after = response_data.get("next_search_after") + if not next_search_after: + break + + if not slice_id: + raise LabellerrError("Could not retrieve slice_id from project files.") + + # 2. Build the new payload matching the UI API exactly + unique_id = client_utils.generate_request_id() + payload = { + "export_name": export_config.export_name, + "export_description": export_config.export_description or "", + "export_format": export_config.export_format, + "connection_id": export_config.connection_id, + "destination": export_config.export_destination.value, + "export_all": False, + "export_status": "Generating Export", + "file_activity": False, + "file_ids": file_ids, + "file_names": [], + "question_ids": export_config.question_ids or ["all"], + "slice_id": slice_id + } + + # 3. Call the UI exports endpoint appending client_id in query parameters + url = f"{constants.BASE_URL}/exports/files?project_id={self.project_id}&client_id={self.client.client_id}&uuid={unique_id}" + + headers = { + "Origin": constants.ALLOWED_ORIGINS, + "Content-Type": "application/json", + } + + response = self.client.make_request( + "POST", + url, + extra_headers=headers, + request_id=unique_id, + data=json.dumps(payload), + ) + # Robustly parse the report_id from the response (which could be a dict or a raw string/JSON string) + if isinstance(response, str): + try: + parsed = json.loads(response) + if isinstance(parsed, dict): + report_id = parsed.get("response", {}).get("report_id") or parsed.get("report_id") or response + else: + report_id = str(parsed) + except ValueError: + report_id = response + elif isinstance(response, dict): + res_val = response.get("response") + if isinstance(res_val, dict): + report_id = res_val.get("report_id") or response.get("report_id") + elif isinstance(res_val, str): + report_id = res_val + else: + report_id = response.get("report_id") or str(response) + else: + report_id = str(response) + + return Export(report_id=report_id, project=self) + def create_export(self, export_config: schemas.CreateExportParams): """ Creates an export with the given configuration. @@ -453,23 +540,9 @@ def create_export(self, export_config: schemas.CreateExportParams): return self.create_local_export(export_config) else: - payload = export_config.model_dump() if not export_config.connection_id or export_config.connection_id == "": raise LabellerrError("connection_id is required") - payload.update( - { - "question_ids": ["all"], - } - ) - - response = self.client.make_request( - "POST", - f"{constants.BASE_URL}/sdk/export/files?project_id={self.project_id}&client_id={self.client.client_id}", - extra_headers={"Content-Type": "application/json"}, - data=json.dumps(payload), - ) - report_id = response.get("response", {}).get("report_id") - return Export(report_id=report_id, project=self) + return self.__execute_ui_style_export(export_config) def create_local_export(self, export_config: schemas.CreateExportParams): """ @@ -479,28 +552,7 @@ def create_local_export(self, export_config: schemas.CreateExportParams): :return: Export instance with report_id and status tracking :raises LabellerrError: If the export creation fails """ - - unique_id = client_utils.generate_request_id() - - export_config_dict = export_config.model_dump() - export_config_dict.update( - {"export_destination": schemas.ExportDestination.LOCAL.value} - ) - - payload = json.dumps(export_config_dict) - - response = self.client.make_request( - "POST", - f"{constants.BASE_URL}/sdk/export/files?project_id={self.project_id}&client_id={self.client.client_id}", - extra_headers={ - "Origin": constants.ALLOWED_ORIGINS, - "Content-Type": "application/json", - }, - request_id=unique_id, - data=payload, - ) - report_id = response.get("response", {}).get("report_id") - return Export(report_id=report_id, project=self) + return self.__execute_ui_style_export(export_config) def list_exports(self): """ diff --git a/labellerr/core/schemas.py b/labellerr/core/schemas.py index 83766fb..00a8d71 100644 --- a/labellerr/core/schemas.py +++ b/labellerr/core/schemas.py @@ -340,7 +340,7 @@ class ListFileParams(BaseModel): client_id: str = Field(min_length=1) project_id: str = Field(min_length=1) - search_queries: Dict[str, Any] + search_queries: Any size: int = Field(default=10, gt=0) next_search_after: Optional[Any] = None diff --git a/labellerr/core/schemas/files.py b/labellerr/core/schemas/files.py index 6e55c21..8b5eb65 100644 --- a/labellerr/core/schemas/files.py +++ b/labellerr/core/schemas/files.py @@ -12,7 +12,7 @@ class ListFileParams(BaseModel): client_id: str = Field(min_length=1) project_id: str = Field(min_length=1) - search_queries: Dict[str, Any] + search_queries: Any size: int = Field(default=10, gt=0) next_search_after: Optional[Any] = None From f3baaf1900210819fabd9e4520ce61679f14bc32 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:35:11 +0530 Subject: [PATCH 02/10] fixed create template --- labellerr/core/annotation_templates/__init__.py | 2 +- tests/unit/test_template_creation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labellerr/core/annotation_templates/__init__.py b/labellerr/core/annotation_templates/__init__.py index 82a607a..f26c74f 100644 --- a/labellerr/core/annotation_templates/__init__.py +++ b/labellerr/core/annotation_templates/__init__.py @@ -52,7 +52,7 @@ def create_template( question_dict.pop("question_type", None) questions_data.append(question_dict) - payload = {"templateName": params.template_name, "questions": questions_data} + payload = {"template_name": params.template_name, "questions": questions_data} url = ( f"{constants.BASE_URL}/annotations/create_template?client_id={client.client_id}&data_type={params.data_type.value}" f"&uuid={unique_id}" diff --git a/tests/unit/test_template_creation.py b/tests/unit/test_template_creation.py index 71007dc..ee56b28 100644 --- a/tests/unit/test_template_creation.py +++ b/tests/unit/test_template_creation.py @@ -78,7 +78,7 @@ def test_create_template_single_bbox_question(self, mock_client): # Verify the create request payload create_call = mock_request.call_args_list[0] payload = create_call[1]["json"] - assert payload["templateName"] == "Single BBox Template" + assert payload["template_name"] == "Single BBox Template" assert len(payload["questions"]) == 1 assert payload["questions"][0]["option_type"] == "BoundingBox" assert payload["questions"][0]["color"] == "#FF0000" From 33ec57be7363da7478959542e27335e9c3031155 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:35:56 +0530 Subject: [PATCH 03/10] added test file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9db31ac..e521182 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ htmlcov/ .pytest_cache/ *.xml *.html + +test.py +test2.py From 9f346560685f1aea179a98564a0dbe5240708cea Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:42:37 +0530 Subject: [PATCH 04/10] Fixed lint error --- labellerr/core/projects/base.py | 4 ++-- labellerr/core/schemas/files.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index f0cb85b..825d318 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -451,7 +451,7 @@ def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): while True: files_res = self.list_files(search_queries=[], size=1000, next_search_after=next_search_after) response_data = files_res.get("response", {}) - + if first_page: slice_id = response_data.get("slice_id") first_page = False @@ -492,7 +492,7 @@ def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): # 3. Call the UI exports endpoint appending client_id in query parameters url = f"{constants.BASE_URL}/exports/files?project_id={self.project_id}&client_id={self.client.client_id}&uuid={unique_id}" - + headers = { "Origin": constants.ALLOWED_ORIGINS, "Content-Type": "application/json", diff --git a/labellerr/core/schemas/files.py b/labellerr/core/schemas/files.py index 8b5eb65..f9ccea7 100644 --- a/labellerr/core/schemas/files.py +++ b/labellerr/core/schemas/files.py @@ -2,7 +2,7 @@ Schema models for file operations. """ -from typing import Any, Dict, List, Optional +from typing import Any, List, Optional from pydantic import BaseModel, Field From bd7ce0ef63908e4a785f45e710d697bc669975c8 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:46:22 +0530 Subject: [PATCH 05/10] fixed unit test --- labellerr/core/schemas.py | 4 ++-- labellerr/core/schemas/files.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/labellerr/core/schemas.py b/labellerr/core/schemas.py index 00a8d71..33ba4a6 100644 --- a/labellerr/core/schemas.py +++ b/labellerr/core/schemas.py @@ -4,7 +4,7 @@ import os from enum import Enum -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID from pydantic import BaseModel, Field, field_validator @@ -340,7 +340,7 @@ class ListFileParams(BaseModel): client_id: str = Field(min_length=1) project_id: str = Field(min_length=1) - search_queries: Any + search_queries: Union[List[Any], Dict[str, Any]] size: int = Field(default=10, gt=0) next_search_after: Optional[Any] = None diff --git a/labellerr/core/schemas/files.py b/labellerr/core/schemas/files.py index f9ccea7..3eca21f 100644 --- a/labellerr/core/schemas/files.py +++ b/labellerr/core/schemas/files.py @@ -2,7 +2,7 @@ Schema models for file operations. """ -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel, Field @@ -12,7 +12,7 @@ class ListFileParams(BaseModel): client_id: str = Field(min_length=1) project_id: str = Field(min_length=1) - search_queries: Any + search_queries: Union[List[Any], Dict[str, Any]] size: int = Field(default=10, gt=0) next_search_after: Optional[Any] = None From b3a0c121dd8246380665deedab7e25fdca08d5eb Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:09:03 +0530 Subject: [PATCH 06/10] refactore export api --- labellerr/core/projects/base.py | 89 +++++++++++---------------------- 1 file changed, 30 insertions(+), 59 deletions(-) diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index 825d318..7d30116 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -441,39 +441,30 @@ def upload_preannotations( else: return future.result() - def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): - # 1. Fetch all files from project to get slice_id and file_ids - file_ids = [] - slice_id = None - next_search_after = None - first_page = True - - while True: - files_res = self.list_files(search_queries=[], size=1000, next_search_after=next_search_after) - response_data = files_res.get("response", {}) - - if first_page: - slice_id = response_data.get("slice_id") - first_page = False - - files_list = response_data.get("files", []) - if not files_list: - break - - for file_item in files_list: - file_status = file_item.get("status") - # Filter by status if statuses are provided - if not export_config.statuses or file_status in export_config.statuses: - file_ids.append(file_item.get("file_id")) - - next_search_after = response_data.get("next_search_after") - if not next_search_after: - break - + def _build_export_search_queries(self, export_config: schemas.CreateExportParams) -> list: + search_queries = [] + # Filter out 'None' string — backend search API doesn't support null status filter + valid_statuses = [s for s in (export_config.statuses or []) if s and s != 'None'] + if valid_statuses: + search_queries.append({"id": "status", "values": valid_statuses}) + if export_config.updated_after_timestamp: + search_queries.append({"id": "updated_after_timestamp", "values": export_config.updated_after_timestamp}) + return search_queries + + def _fetch_slice_id(self, search_queries: list) -> str: + files_res = self.list_files(search_queries=search_queries, size=10) + response_data = files_res.get("response", {}) + slice_id = response_data.get("slice_id") if not slice_id: raise LabellerrError("Could not retrieve slice_id from project files.") + return slice_id + + def _submit_export_job(self, export_config: schemas.CreateExportParams): + # 1. Get slice_id based on user search filters + search_queries = self._build_export_search_queries(export_config) + slice_id = self._fetch_slice_id(search_queries) - # 2. Build the new payload matching the UI API exactly + # 2. Build the payload unique_id = client_utils.generate_request_id() payload = { "export_name": export_config.export_name, @@ -481,18 +472,16 @@ def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): "export_format": export_config.export_format, "connection_id": export_config.connection_id, "destination": export_config.export_destination.value, - "export_all": False, + "export_all": True, "export_status": "Generating Export", "file_activity": False, - "file_ids": file_ids, - "file_names": [], "question_ids": export_config.question_ids or ["all"], - "slice_id": slice_id + "slice_id": slice_id, + "export_path": export_config.export_folder_path, } - # 3. Call the UI exports endpoint appending client_id in query parameters + # 3. Call the UI exports endpoint url = f"{constants.BASE_URL}/exports/files?project_id={self.project_id}&client_id={self.client.client_id}&uuid={unique_id}" - headers = { "Origin": constants.ALLOWED_ORIGINS, "Content-Type": "application/json", @@ -505,29 +494,12 @@ def __execute_ui_style_export(self, export_config: schemas.CreateExportParams): request_id=unique_id, data=json.dumps(payload), ) - # Robustly parse the report_id from the response (which could be a dict or a raw string/JSON string) - if isinstance(response, str): - try: - parsed = json.loads(response) - if isinstance(parsed, dict): - report_id = parsed.get("response", {}).get("report_id") or parsed.get("report_id") or response - else: - report_id = str(parsed) - except ValueError: - report_id = response - elif isinstance(response, dict): - res_val = response.get("response") - if isinstance(res_val, dict): - report_id = res_val.get("report_id") or response.get("report_id") - elif isinstance(res_val, str): - report_id = res_val - else: - report_id = response.get("report_id") or str(response) - else: - report_id = str(response) + # 4. Extract report_id from response + report_id = response.get("response") if isinstance(response, dict) else response return Export(report_id=report_id, project=self) + def create_export(self, export_config: schemas.CreateExportParams): """ Creates an export with the given configuration. @@ -538,11 +510,10 @@ def create_export(self, export_config: schemas.CreateExportParams): """ if export_config.export_destination == schemas.ExportDestination.LOCAL: return self.create_local_export(export_config) - else: if not export_config.connection_id or export_config.connection_id == "": raise LabellerrError("connection_id is required") - return self.__execute_ui_style_export(export_config) + return self._submit_export_job(export_config) def create_local_export(self, export_config: schemas.CreateExportParams): """ @@ -552,7 +523,7 @@ def create_local_export(self, export_config: schemas.CreateExportParams): :return: Export instance with report_id and status tracking :raises LabellerrError: If the export creation fails """ - return self.__execute_ui_style_export(export_config) + return self._submit_export_job(export_config) def list_exports(self): """ From 6c3e5d6390c12ab2b91bd271056e5d109afad3f5 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:11:16 +0530 Subject: [PATCH 07/10] added git ignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e521182..eb03ef8 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ htmlcov/ test.py test2.py +test_.py +test_list_files_format.py + From ce90373ae07872293e94d4fc1bbf0cc4665c4e87 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:49:32 +0530 Subject: [PATCH 08/10] remove none checking --- .gitignore | 2 ++ labellerr/core/projects/base.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index eb03ef8..1cddd92 100644 --- a/.gitignore +++ b/.gitignore @@ -46,4 +46,6 @@ test.py test2.py test_.py test_list_files_format.py +testviatimestamp.py + diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index 7d30116..1480fab 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -443,8 +443,7 @@ def upload_preannotations( def _build_export_search_queries(self, export_config: schemas.CreateExportParams) -> list: search_queries = [] - # Filter out 'None' string — backend search API doesn't support null status filter - valid_statuses = [s for s in (export_config.statuses or []) if s and s != 'None'] + valid_statuses = [s for s in (export_config.statuses or []) if s] if valid_statuses: search_queries.append({"id": "status", "values": valid_statuses}) if export_config.updated_after_timestamp: From ae7b418f36664d6c5728ec21065d24e9fc39003e Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:51:18 +0530 Subject: [PATCH 09/10] fixed linting error --- labellerr/core/projects/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index 1480fab..7926cc0 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -498,7 +498,6 @@ def _submit_export_job(self, export_config: schemas.CreateExportParams): report_id = response.get("response") if isinstance(response, dict) else response return Export(report_id=report_id, project=self) - def create_export(self, export_config: schemas.CreateExportParams): """ Creates an export with the given configuration. From c788d72a292e5c4ec939ae5b0ff02b09dcb3bfc0 Mon Sep 17 00:00:00 2001 From: Akash rawal <158574393+akash-rawal@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:55:17 +0530 Subject: [PATCH 10/10] fixed the api --- labellerr/core/projects/base.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/labellerr/core/projects/base.py b/labellerr/core/projects/base.py index 7926cc0..b538824 100644 --- a/labellerr/core/projects/base.py +++ b/labellerr/core/projects/base.py @@ -442,12 +442,22 @@ def upload_preannotations( return future.result() def _build_export_search_queries(self, export_config: schemas.CreateExportParams) -> list: + import time search_queries = [] valid_statuses = [s for s in (export_config.statuses or []) if s] if valid_statuses: - search_queries.append({"id": "status", "values": valid_statuses}) + search_queries.append({ + "op": "OR", + "id": "file_status", + "values": [{"p": "in", "v": valid_statuses}], + }) if export_config.updated_after_timestamp: - search_queries.append({"id": "updated_after_timestamp", "values": export_config.updated_after_timestamp}) + now_ms = int(time.time() * 1000) + search_queries.append({ + "op": "OR", + "id": "last_updated_date", + "values": [{"p": "between", "v": [{"start": export_config.updated_after_timestamp, "end": now_ms}]}], + }) return search_queries def _fetch_slice_id(self, search_queries: list) -> str: