diff --git a/.gitignore b/.gitignore index 9cdce73..daf460e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,8 @@ wheels/ .env .DS_Store .claude -tests/test_data \ No newline at end of file + +# Test data +tests/test_data +download +labellerr/__pycache__/ diff --git a/labellerr/base/singleton.py b/labellerr/base/singleton.py index a4c429e..93fc392 100644 --- a/labellerr/base/singleton.py +++ b/labellerr/base/singleton.py @@ -2,18 +2,19 @@ class Singleton: - __instance = None - __lock = None + _instances = {} + _locks = {} def __new__(cls, *args, **kwargs): - if cls.__lock is None: - cls.__lock = threading.Lock() - if cls.__instance is None: - with cls.__lock: - if cls.__instance is None: - cls.__instance = super().__new__(cls) - return cls.__instance + if cls not in cls._locks: + cls._locks[cls] = threading.Lock() + + if cls not in cls._instances: + with cls._locks[cls]: + if cls not in cls._instances: + cls._instances[cls] = super().__new__(cls) + return cls._instances[cls] def __init__(self, *args): if type(self) is Singleton: - raise TypeError("Can't instantiate Singleton class") + raise TypeError("Can't instantiate Singleton class") \ No newline at end of file diff --git a/labellerr/client.py b/labellerr/client.py index 87db582..21fd183 100644 --- a/labellerr/client.py +++ b/labellerr/client.py @@ -71,6 +71,7 @@ def __init__( self, api_key, api_secret, + client_id, enable_connection_pooling=True, pool_connections=10, pool_maxsize=20, @@ -80,12 +81,14 @@ def __init__( :param api_key: The API key for authentication. :param api_secret: The API secret for authentication. + :param client_id: The client ID for the Labellerr account. :param enable_connection_pooling: Whether to enable connection pooling :param pool_connections: Number of connection pools to cache :param pool_maxsize: Maximum number of connections to save in the pool """ self.api_key = api_key self.api_secret = api_secret + self.client_id = client_id self.base_url = constants.BASE_URL self._session = None self._enable_pooling = enable_connection_pooling diff --git a/labellerr/core/datasets/__init__.py b/labellerr/core/datasets/__init__.py index 6b7b5b1..5641a6c 100644 --- a/labellerr/core/datasets/__init__.py +++ b/labellerr/core/datasets/__init__.py @@ -1,2 +1,7 @@ """This module will contain all CRUD for datasets. Example, create, list datasets, get dataset, delete dataset, update dataset, etc. """ +from labellerr.core.datasets.base import LabellerrDataset + +__all__ = [ + 'LabellerrDataset' + ] diff --git a/labellerr/core/datasets/base.py b/labellerr/core/datasets/base.py new file mode 100644 index 0000000..c270061 --- /dev/null +++ b/labellerr/core/datasets/base.py @@ -0,0 +1,180 @@ +from labellerr.client import LabellerrClient +from labellerr.exceptions import LabellerrError +from labellerr.core.files import LabellerrFile +from labellerr import constants +import uuid +from abc import ABCMeta +import pprint + +class LabellerrDataset: + """ + Class for handling video dataset operations and fetching multiple video files. + """ + + def __init__(self, client: LabellerrClient, dataset_id: str, project_id: str): + """ + Initialize video dataset instance. + + :param client: LabellerrClient instance + :param dataset_id: Dataset ID + :param project_id: Project ID containing the dataset + """ + self.client = client + self.dataset_id = dataset_id + self.project_id = project_id + self.client_id = client.client_id + + def fetch_files(self, page_size: int = 1000): + """ + Fetch all video files in this dataset as LabellerrVideoFile instances. + + :param page_size: Number of files to fetch per API request (default: 10) + :return: List of file IDs + """ + try: + all_file_ids = [] + next_search_after = None # Start with None for first page + + while True: + unique_id = str(uuid.uuid4()) + url = f"{constants.BASE_URL}/search/files/all" + params = { + 'sort_by': 'created_at', + 'sort_order': 'desc', + 'size': page_size, + 'uuid': unique_id, + 'dataset_id': self.dataset_id, + 'client_id': self.client_id + } + + # Add next_search_after only if it exists (don't send on first request) + if next_search_after: + url+= f"?next_search_after={next_search_after}" + + # print(params) + + response = self.client.make_api_request(self.client_id, url, params, unique_id) + + # pprint.pprint(response) + + # Extract files from the response + files = response.get('response', {}).get('files', []) + + # Collect file IDs + for file_info in files: + file_id = file_info.get('file_id') + if file_id: + all_file_ids.append(file_id) + + # Get next_search_after for pagination + next_search_after = response.get('response', {}).get('next_search_after') + + + # Break if no more pages or no files returned + if not next_search_after or not files: + break + + print(f"Fetched total: {len(all_file_ids)}") + + print(f"Total file IDs extracted: {len(all_file_ids)}") + # return all_file_ids + + # Create LabellerrVideoFile instances for each file_id + video_files = [] + print(f"\nCreating LabellerrFile instances for {len(all_file_ids)} files...") + + for file_id in all_file_ids: + try: + video_file = LabellerrFile( + client=self.client, + file_id=file_id, + project_id=self.project_id, + dataset_id=self.dataset_id + ) + video_files.append(video_file) + except Exception as e: + print(f"Warning: Failed to create file instance for {file_id}: {str(e)}") + + print(f"Successfully created {len(video_files)} LabellerrFile instances") + return video_files + + except Exception as e: + raise LabellerrError(f"Failed to fetch dataset files: {str(e)}") + + def download(self): + """ + Process all video files in the dataset: download frames, create videos, + and automatically clean up temporary files. + + :param output_folder: Base folder where dataset folder will be created + :return: List of processing results for all files + """ + try: + print(f"\n{'#'*70}") + print(f"# Starting batch video processing for dataset: {self.dataset_id}") + print(f"{'#'*70}\n") + + # Fetch all video files + video_files = self.fetch_files() + + if not video_files: + print("No video files found in dataset") + return [] + + print(f"\nProcessing {len(video_files)} video files...\n") + + results = [] + successful = 0 + failed = 0 + + print(f"\nStarting download of {len(video_files)} files...") + for idx, video_file in enumerate(video_files, 1): + try: + # Call the new all-in-one method + result = video_file.download_create_video_auto_cleanup() + results.append(result) + successful += 1 + print(f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)", end="", flush=True) + + except Exception as e: + error_result = { + 'status': 'failed', + 'file_id': video_file.file_id, + 'error': str(e) + } + results.append(error_result) + failed += 1 + print(f"\rFiles processed: {idx}/{len(video_files)} ({successful} successful, {failed} failed)", end="", flush=True) + + # Summary + print(f"\n{'#'*70}") + print(f"# Batch Processing Complete") + print(f"# Total files: {len(video_files)}") + print(f"# Successful: {successful}") + print(f"# Failed: {failed}") + print(f"{'#'*70}\n") + + return results + + except Exception as e: + raise LabellerrError(f"Failed to process dataset videos: {str(e)}") + + +# if __name__ == "__main__": +# # Example usage +# api_key = "" +# api_secret = "" +# client_id = "" + +# dataset_id = "59438ec3-12e0-4687-8847-1e6e01b0bf25" +# project_id = "farrah_supposed_hookworm_34155" + +# client = LabellerrClient(api_key, api_secret, client_id) + +# dataset = LabellerrVideoDataset(client, dataset_id, project_id) + +# # Process all videos in the dataset +# results = dataset.download() + +# # Print summary +# pprint.pprint(results) \ No newline at end of file diff --git a/labellerr/core/files/__init__.py b/labellerr/core/files/__init__.py new file mode 100644 index 0000000..51a721f --- /dev/null +++ b/labellerr/core/files/__init__.py @@ -0,0 +1,14 @@ +# Import base classes +from labellerr.core.files.base import LabellerrFile, LabellerrFileMeta + +# Import subclasses to trigger registration +# These imports register each file type with the metaclass +from labellerr.core.files.image_file import LabellerrImageFile +from labellerr.core.files.video_file import LabellerrVideoFile + +__all__ = [ + 'LabellerrFile', + 'LabellerrImageFile', + 'LabellerrVideoFile', + 'LabellerrFileMeta' +] \ No newline at end of file diff --git a/labellerr/core/files/base.py b/labellerr/core/files/base.py new file mode 100644 index 0000000..c62dcc9 --- /dev/null +++ b/labellerr/core/files/base.py @@ -0,0 +1,131 @@ +from labellerr.client import LabellerrClient +from labellerr.exceptions import LabellerrError +from labellerr import constants +import uuid +from abc import ABCMeta + + +class LabellerrFileMeta(ABCMeta): + """Metaclass that combines ABC functionality with factory pattern""" + + _registry = {} + + @classmethod + def register(cls, data_type, file_class): + """Register a file type handler""" + cls._registry[data_type.lower()] = file_class + + + def __call__(cls, client, file_id, project_id, dataset_id = None, **kwargs): + + if cls.__name__ != 'LabellerrFile': + + instance = cls.__new__(cls) + if isinstance(instance, cls): + instance.__init__(client, file_id, project_id, dataset_id=dataset_id, **kwargs) + return instance + + + try: + unique_id = str(uuid.uuid4()) + client_id = client.client_id + params = { + 'file_id': file_id, + 'include_answers': 'false', + 'project_id': project_id, + 'uuid': unique_id, + 'client_id': client_id + } + + # TODO: Add dataset_id to params based on precedence logic + # Priority: project_id > dataset_id + + url = f"{constants.BASE_URL}/data/file_data" + response = client.make_api_request(client_id, url, params, unique_id) + + # Extract data_type from response + file_metadata = response.get('file_metadata', {}) + data_type = response.get('data_type', '').lower() + + # print(f"Detected file type: {data_type}") + + file_class = cls._registry.get(data_type) + if file_class is None: + raise LabellerrError(f"Unsupported file type: {data_type}") + + return file_class(client, file_id, project_id, dataset_id=dataset_id, file_metadata=file_metadata) + + except Exception as e: + raise LabellerrError(f"Failed to create file instance: {str(e)}") + + + + # # Route to appropriate subclass + # if data_type == 'image': + # return LabellerrImageFile(client, file_id, project_id, dataset_id=dataset_id, + # file_metadata=file_metadata) + # elif data_type == 'video': + # return LabellerrVideoFile(client, file_id, project_id, dataset_id=dataset_id, + # file_metadata=file_metadata) + # else: + # raise LabellerrError(f"Unsupported file type: {data_type}") + + + # except Exception as e: + # raise LabellerrError(f"Failed to create file instance: {str(e)}") + + +class LabellerrFile(metaclass=LabellerrFileMeta): + """Base class for all Labellerr files with factory behavior""" + + def __init__(self, client: LabellerrClient, file_id: str, project_id: str, + dataset_id: str | None = None, **kwargs): + """ + Initialize base file attributes + + :param client: LabellerrClient instance + :param file_id: Unique file identifier + :param project_id: Project ID containing the file + :param dataset_id: Optional dataset ID + :param kwargs: Additional file data (file_metadata, response, etc.) + """ + self.client = client + self.file_id = file_id + self.project_id = project_id + self.client_id = client.client_id + self.dataset_id = dataset_id + + # Store metadata from factory creation + self.metadata = kwargs.get('file_metadata', {}) + + + def get_metadata(self, include_answers: bool = False): + """ + Refresh and retrieve file metadata from Labellerr API. + + :param include_answers: Whether to include annotation answers + :return: Dictionary containing file metadata + """ + try: + unique_id = str(uuid.uuid4()) + + params = { + 'file_id': self.file_id, + 'include_answers': str(include_answers).lower(), + 'project_id': self.project_id, + 'uuid': unique_id, + 'client_id': self.client_id + } + + # TODO: Add dataset_id handling if needed + + url = f"{constants.BASE_URL}/data/file_data" + response = self.client.make_api_request(self.client_id, url, params, unique_id) + + # Update cached metadata + self.metadata = response.get('file_metadata', {}) + + return response + + except Exception as e: + raise LabellerrError(f"Failed to fetch file metadata: {str(e)}") diff --git a/labellerr/core/files/image_file.py b/labellerr/core/files/image_file.py new file mode 100644 index 0000000..1e66938 --- /dev/null +++ b/labellerr/core/files/image_file.py @@ -0,0 +1,4 @@ +from labellerr.core.files.base import LabellerrFile + +class LabellerrImageFile(LabellerrFile): + pass \ No newline at end of file diff --git a/labellerr/core/files/video_file.py b/labellerr/core/files/video_file.py new file mode 100644 index 0000000..605b5c0 --- /dev/null +++ b/labellerr/core/files/video_file.py @@ -0,0 +1,311 @@ +from labellerr.client import LabellerrClient +from labellerr.exceptions import LabellerrError +from labellerr import constants +import uuid +import os +import subprocess +import requests +import shutil +from concurrent.futures import ThreadPoolExecutor, as_completed +from threading import Lock +from labellerr.core.files.base import LabellerrFile, LabellerrFileMeta + +class LabellerrVideoFile(LabellerrFile): + """Specialized class for handling video files including frame operations""" + + def __init__(self, client: LabellerrClient, file_id: str, project_id: str, dataset_id: str | None = None, **kwargs): + super().__init__(client, file_id, project_id, dataset_id=dataset_id, **kwargs) + + @property + def total_frames(self): + """Get total number of frames in the video.""" + return self.metadata.get('total_frames', 0) + + def get_frames(self, frame_start: int = 0, frame_end: int | None = None): + """ + Retrieve video frames data from Labellerr API. + + :param frame_start: Starting frame index (default: 0) + :param frame_end: Ending frame index (default: total_frames) + :return: Dictionary containing video frames data with frame numbers as keys and URLs as values + """ + try: + if self.dataset_id is None: + raise ValueError("dataset_id is required for fetching video frames") + + # Use total_frames as default for frame_end + if frame_end is None: + frame_end = self.total_frames + + unique_id = str(uuid.uuid4()) + url = f"{constants.BASE_URL}/data/video_frames" + + params = { + 'dataset_id': self.dataset_id, + 'file_id': self.file_id, + 'frame_start': frame_start, + 'frame_end': frame_end, + 'project_id': self.project_id, + 'uuid': unique_id, + 'client_id': self.client_id + } + + response = self.client.make_api_request(self.client_id, url, params, unique_id) + + return response + + except Exception as e: + raise LabellerrError(f"Failed to fetch video frames data: {str(e)}") + + def _download_single_frame(self, frame_number, frame_url, save_path, print_lock): + """ + Download a single frame (helper method for threading). + + :param frame_number: Frame number + :param frame_url: URL to download from + :param save_path: Directory to save the frame + :param print_lock: Lock for thread-safe printing + :return: Tuple of (success: bool, frame_number: str, error_info: dict or None) + """ + try: + filename = f"{frame_number}.jpg" + filepath = os.path.join(save_path, filename) + + response = requests.get(frame_url, timeout=30) + + if response.status_code == 200: + with open(filepath, 'wb') as f: + f.write(response.content) + return True, frame_number, None + else: + error_info = { + 'frame': frame_number, + 'status': response.status_code + } + return False, frame_number, error_info + + except Exception as e: + error_info = { + 'frame': frame_number, + 'error': str(e) + } + with print_lock: + print(f"Error downloading frame {frame_number}: {str(e)}") + + return False, frame_number, error_info + + def download_frames(self, frames_data: dict, output_folder: str | None = None, + max_workers: int = 30): + """ + Download video frames from URLs to a local folder using multithreading. + + :param frames_data: Dictionary with frame numbers as keys and URLs as values + :param output_folder: Base folder path where frames will be saved (default: current directory) + :param max_workers: Maximum number of concurrent download threads (default: 10) + :return: Dictionary with download statistics + """ + try: + # Use file_id as folder name + folder_name = self.file_id + + # Set output path + if output_folder: + save_path = os.path.join(output_folder, folder_name) + else: + save_path = folder_name + + # Create directory if it doesn't exist + os.makedirs(save_path, exist_ok=True) + + success_count = 0 + failed_frames = [] + print_lock = Lock() + total_frames = len(frames_data) + + print(f"Starting download of {total_frames} frames...") + + # Use ThreadPoolExecutor for concurrent downloads + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all download tasks + future_to_frame = { + executor.submit( + self._download_single_frame, + frame_number, + frame_url, + save_path, + print_lock + ): frame_number + for frame_number, frame_url in frames_data.items() + } + + completed = 0 + # Process completed downloads + for future in as_completed(future_to_frame): + success, frame_number, error_info = future.result() + completed += 1 + + if success: + success_count += 1 + else: + failed_frames.append(error_info) + + # Update progress + with print_lock: + print(f"\rFrames downloaded: {completed}/{total_frames} ({success_count} successful, {len(failed_frames)} failed)", end="", flush=True) + + # Print newline after progress + print() + + result = { + 'file_id': self.file_id, + 'total_frames': total_frames, + 'successful_downloads': success_count, + 'failed_downloads': len(failed_frames), + 'save_path': save_path, + 'failed_frames': failed_frames + } + + # print(f"\nDownload complete: {success_count}/{len(frames_data)} frames downloaded successfully") + + return result + + except Exception as e: + raise LabellerrError(f"Failed to download video frames: {str(e)}") + + def create_video(self, frames_folder: str, + framerate: int = 30, pattern: str = "%d.jpg", output_file: str | None = None): + """ + Join frames into a video using ffmpeg. + + :param frames_folder: Path to folder containing sequential frames (e.g., 1.jpg, 2.jpg). + :param output_file: Name of the output video file (default: output.mp4). + :param framerate: Desired video framerate (default: 30 fps). + :param pattern: Pattern for sequential frames (default: %d.jpg → 1.jpg, 2.jpg, ...). + :return: Path to created video file + """ + if frames_folder is None: + raise ValueError("frames_folder must be provided") + + input_pattern = os.path.join(frames_folder, pattern) + if output_file is None: + output_file = f"{self.file_id}.mp4" + + # FFmpeg command + command = [ + "ffmpeg", + "-y", # Overwrite output file if exists + "-start_number", "0", + "-framerate", str(framerate), + "-i", input_pattern, + "-c:v", "libx264", + "-pix_fmt", "yuv420p", + output_file + ] + + try: + print("Running command:", " ".join(command)) + subprocess.run(command, check=True) + print(f"Video saved as {output_file}") + return output_file + except subprocess.CalledProcessError as e: + raise LabellerrError(f"Error while joining frames: {str(e)}") + + def download_create_video_auto_cleanup(self, output_folder: str = "./Labellerr_datastets"): + """ + Download frames, create video, and automatically clean up temporary frames. + This is an all-in-one method for processing video files. + Downloads all frames from 0 to total_frames automatically. + + :return: Dictionary with operation results + """ + try: + print(f"\n{'='*60}") + print(f"Processing file: {self.file_id}") + print(f"{'='*60}") + + # Step 1: Get total frames + total_frames = self.total_frames + if total_frames == 0: + raise LabellerrError("No frames found for this video file") + + # Step 2: Fetch frame data from API + print(f"\n[1/4] Fetching frame data from API (0 to {total_frames})...") + frames_data = self.get_frames(frame_start=0, frame_end=total_frames) + + if not frames_data: + raise LabellerrError("No frame data retrieved from API") + + print(f"Retrieved {len(frames_data)} frames") + + # Step 2: Create dataset folder structure + print(f"\n[2/4] Setting up output folders...") + if self.dataset_id is None: + dataset_folder = output_folder + else: + dataset_folder = os.path.join(output_folder, self.dataset_id) + os.makedirs(dataset_folder, exist_ok=True) + + # Define actual frames folder path + actual_frames_folder = os.path.join(dataset_folder, self.file_id) + + # Step 3: Download frames + print(f"\n[3/4] Downloading frames...") + download_result = self.download_frames( + frames_data=frames_data, + output_folder=dataset_folder + ) + + if download_result['failed_downloads'] > 0: + print(f"\nWarning: {download_result['failed_downloads']} frames failed to download") + + # Step 4: Create video from downloaded frames + print(f"\n[4/4] Creating video from frames...") + video_output_path = os.path.join(dataset_folder, f"{self.file_id}.mp4") + + self.create_video( + frames_folder=actual_frames_folder, + output_file=video_output_path + ) + + # Step 5: Clean up temporary frames folder + print(f"\nCleaning up temporary frames...") + if os.path.exists(actual_frames_folder): + shutil.rmtree(actual_frames_folder) + print(f"Removed temporary frames folder: {actual_frames_folder}") + + result = { + 'status': 'success', + 'file_id': self.file_id, + 'dataset_id': self.dataset_id, + 'video_path': video_output_path, + 'output_folder': dataset_folder, + 'frames_downloaded': download_result['successful_downloads'], + 'frames_failed': download_result['failed_downloads'], + 'failed_frames_info': download_result['failed_frames'] + } + + print(f"\n{'='*60}") + print(f"✓ Processing complete!") + print(f"Video saved to: {video_output_path}") + print(f"{'='*60}\n") + + return result + + except Exception as e: + # Attempt cleanup on error + try: + # Get the frames folder path + if self.dataset_id is None: + cleanup_folder = os.path.join(output_folder, self.file_id) + else: + cleanup_folder = os.path.join(output_folder, self.dataset_id, self.file_id) + + if os.path.exists(cleanup_folder): + shutil.rmtree(cleanup_folder) + except: + pass + + raise LabellerrError(f"Failed in video processing: {str(e)}") + + +LabellerrFileMeta.register('video', LabellerrVideoFile) \ No newline at end of file diff --git a/labellerr/notebooks/SDK.ipynb b/labellerr/notebooks/SDK.ipynb new file mode 100644 index 0000000..1efd28f --- /dev/null +++ b/labellerr/notebooks/SDK.ipynb @@ -0,0 +1,730 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d6488b6b", + "metadata": {}, + "source": [ + "# Getting Started with Labellerr SDK\n", + "\n", + "This notebook demonstrates how to use the Labellerr SDK for video processing and scene detection. The SDK provides powerful tools for managing video datasets, processing videos, and detecting scene changes using various algorithms.\n", + "\n", + "### Import the required Classes from Labellerr SDK\n", + "We'll start by importing the essential classes needed for working with the SDK:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "edcdab6a", + "metadata": {}, + "outputs": [], + "source": [ + "from labellerr.client import LabellerrClient\n", + "from labellerr.core.datasets import LabellerrDataset\n", + "import os\n", + "from tqdm.notebook import tqdm\n" + ] + }, + { + "cell_type": "markdown", + "id": "84b7917a", + "metadata": {}, + "source": [ + "## 1. Authentication Setup\n", + "\n", + "Before using the Labellerr SDK, you need to set up your authentication credentials. These credentials ensure secure access to the Labellerr platform and its services.\n", + "\n", + "### Required Credentials:\n", + "\n", + "1. **API Key & API Secret**\n", + " - Log in to your Labellerr account\n", + " - Navigate to the \"Get API\" tab\n", + " - Copy your unique API key and secret\n", + "\n", + "2. **Client ID**\n", + " - This is a unique identifier for your application\n", + " - Contact Labellerr support to obtain your client ID\n", + " \n", + "⚠️ Important: Never share these credentials or commit them to version control." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ab12f168", + "metadata": {}, + "outputs": [], + "source": [ + "from dotenv import dotenv_values\n", + "config = dotenv_values(\".env\")\n", + "\n", + "api_key = config[\"API_KEY\"]\n", + "api_secret = config[\"API_SECRET\"]\n", + "client_id = config[\"CLIENT_ID\"]" + ] + }, + { + "cell_type": "markdown", + "id": "3d05bd0f", + "metadata": {}, + "source": [ + "## 2. Project Configuration\n", + "\n", + "### Dataset and Project IDs\n", + "To work with specific datasets and projects in Labellerr, you need their respective IDs. These IDs are unique identifiers that link your code to the correct resources on the platform.\n", + "\n", + "How to obtain the IDs:\n", + "1. Go to the Labellerr platform\n", + "2. Create or select an existing dataset\n", + "3. Create or select an existing project\n", + "4. Copy the dataset_id and project_id from their respective pages\n", + "\n", + "Note: The dataset_id is a UUID format string, while the project_id is typically a human-readable string." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "07dcfae9", + "metadata": {}, + "outputs": [], + "source": [ + "# go to our platform to create dataset and project then get their ids\n", + "dataset_id = \"16257fd6-b91b-4d00-a680-9ece9f3f241c\"\n", + "project_id = \"gabrila_artificial_duck_74237\"" + ] + }, + { + "cell_type": "markdown", + "id": "1b2c7aee", + "metadata": {}, + "source": [ + "## 3. Initializing the Labellerr SDK\n", + "\n", + "### Create LabellerrClient Instance\n", + "Now we'll create instances of the main SDK classes:\n", + "\n", + "1. **LabellerrClient**: The main client that handles communication with the Labellerr API\n", + "2. **LabellerrDataset**: A specialized class for working with datasets\n", + "\n", + "These instances will be used for all subsequent operations with the platform." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9eaec7e1", + "metadata": {}, + "outputs": [], + "source": [ + "client = LabellerrClient(api_key, api_secret, client_id) \n", + "dataset = LabellerrDataset(client, dataset_id, project_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7b6a7052", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "######################################################################\n", + "# Starting batch video processing for dataset: 16257fd6-b91b-4d00-a680-9ece9f3f241c\n", + "######################################################################\n", + "\n", + "Total file IDs extracted: 1\n", + "\n", + "Creating LabellerrFile instances for 1 files...\n", + "Successfully created 1 LabellerrFile instances\n", + "\n", + "Processing 1 video files...\n", + "\n", + "\n", + "Starting download of 1 files...\n", + "\n", + "============================================================\n", + "Processing file: c44f38f6-0186-436f-8c2d-ffb50a539c76\n", + "============================================================\n", + "\n", + "[1/4] Fetching frame data from API (0 to 1440)...\n", + "Retrieved 1440 frames\n", + "\n", + "[2/4] Setting up output folders...\n", + "\n", + "[3/4] Downloading frames...\n", + "Starting download of 1440 frames...\n", + "Frames downloaded: 1440/1440 (1440 successful, 0 failed)\n", + "\n", + "[4/4] Creating video from frames...\n", + "Running command: ffmpeg -y -start_number 0 -framerate 30 -i ./Labellerr_datastets\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\%d.jpg -c:v libx264 -pix_fmt yuv420p ./Labellerr_datastets\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76.mp4\n", + "Video saved as ./Labellerr_datastets\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76.mp4\n", + "\n", + "Cleaning up temporary frames...\n", + "Removed temporary frames folder: ./Labellerr_datastets\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76\n", + "\n", + "============================================================\n", + "✓ Processing complete!\n", + "Video saved to: ./Labellerr_datastets\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76.mp4\n", + "============================================================\n", + "\n", + "Files processed: 1/1 (1 successful, 0 failed)\n", + "######################################################################\n", + "# Batch Processing Complete\n", + "# Total files: 1\n", + "# Successful: 1\n", + "# Failed: 0\n", + "######################################################################\n", + "\n" + ] + } + ], + "source": [ + "results = dataset.download()" + ] + }, + { + "cell_type": "markdown", + "id": "900ea5a7", + "metadata": {}, + "source": [ + "### download Videos\n", + "The `download()` method will:\n", + "- Fetch all videos in the dataset\n", + "- Process them according to the configured settings\n", + "- Return the results of the processing\n", + "\n", + "This is typically used as the first step in video analysis to ensure all videos are properly prepared for further processing." + ] + }, + { + "cell_type": "markdown", + "id": "f6db8522", + "metadata": {}, + "source": [ + "## 4. Scene Change Detection\n", + "\n", + "### Available Scene Detection Methods\n", + "Labellerr SDK provides multiple algorithms for scene detection in videos:\n", + "\n", + "1. **PySceneDetect**: \n", + " - Python-based scene detection\n", + " - Uses content-aware detection\n", + " - Good for general-purpose scene detection\n", + "\n", + "2. **SSIMSceneDetect**:\n", + " - Uses Structural Similarity Index (SSIM)\n", + " - Better for detecting subtle scene changes\n", + " - More computationally intensive but more accurate\n", + "\n", + "3. **FFMPEGSceneDetect**:\n", + " - Uses FFMPEG for scene detection\n", + " - Fastest method\n", + " - Good for quick analysis of large video files\n", + "\n", + "Choose the method that best suits your needs based on accuracy requirements and processing speed constraints." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f5c41073", + "metadata": {}, + "outputs": [], + "source": [ + "from labellerr.services.video_sampling.pyscene_detect import PySceneDetect\n", + "from labellerr.services.video_sampling.ssim import SSIMSceneDetect\n", + "from labellerr.services.video_sampling.ffmpeg import FFMPEGSceneDetect" + ] + }, + { + "cell_type": "markdown", + "id": "db88da50", + "metadata": {}, + "source": [ + "## Scene Detection Implementation\n", + "\n", + "### Setting up the Scene Detector\n", + "Now we'll set up the scene detection process:\n", + "\n", + "1. First, we'll define the dataset directory where our videos are stored\n", + "2. Then we'll create an instance of our chosen detector\n", + "3. Finally, we'll process each video in the dataset\n", + "\n", + "Note: Make sure you have sufficient disk space for storing the extracted scenes, as this process can generate multiple files per video." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49a6f89d", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_dir = f\".\\Labellerr_datasets\\{dataset_id}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "dd96be8c", + "metadata": {}, + "outputs": [], + "source": [ + "detector = FFMPEGSceneDetect()" + ] + }, + { + "cell_type": "markdown", + "id": "995d99ec", + "metadata": {}, + "source": [ + "### Initialize the Scene Detector\n", + "Here we create an instance of the SSIMSceneDetect class. This detector uses the Structural Similarity Index Measure (SSIM) to identify scene changes in videos. SSIM is particularly effective at detecting subtle changes between frames." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a3052f25", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Keyframes extracted to FFMPEG_detects\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\frames\n", + "JSON mapping saved to: FFMPEG_detects\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\c44f38f6-0186-436f-8c2d-ffb50a539c76_mapping.json\n" + ] + } + ], + "source": [ + "for filename in os.listdir(dataset_dir):\n", + " file_path = os.path.join(dataset_dir, filename)\n", + " \n", + " if os.path.isfile(file_path):\n", + " detector.detect_and_extract(file_path)" + ] + }, + { + "cell_type": "markdown", + "id": "8d64ac26", + "metadata": {}, + "source": [ + "### Process Videos for Scene Detection\n", + "\n", + "The following code block:\n", + "1. Iterates through all files in the dataset directory\n", + "2. Constructs the full file path for each video\n", + "3. Verifies that each path points to a file (not a directory)\n", + "4. Applies scene detection to each video using the `detect_and_extract` method\n", + "\n", + "The detected scenes will be saved in a subdirectory with the same name as the input video file. Each scene will be saved as a separate video file." + ] + }, + { + "cell_type": "markdown", + "id": "6c3eac46", + "metadata": {}, + "source": [ + "## 5. Project Creation\n", + "\n", + "In this section, we'll explore how to create and manage projects in Labellerr. Projects are essential containers that organize your data and annotations. We'll cover:\n", + "\n", + "1. Creating image datasets from video frames\n", + "2. Setting up annotation projects\n", + "3. Managing project configurations" + ] + }, + { + "cell_type": "markdown", + "id": "f5ba527d", + "metadata": {}, + "source": [ + "### Image Dataset Creation from Sampled Frames\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1b364362", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 52 image files\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "images_files = []\n", + "# Clear existing entries in images_files\n", + "images_files.clear()\n", + "\n", + "# Construct the base directory path for detected frames\n", + "base_dir = os.path.join(\"FFMPEG_detects\", dataset_id)\n", + "\n", + "# Walk through all subdirectories\n", + "for root, dirs, files in os.walk(base_dir):\n", + " for file in files:\n", + " if file.endswith('.jpg'): # Only collect jpg files\n", + " file_path = os.path.join(root, file)\n", + " images_files.append(file_path)\n", + "\n", + "print(f\"Found {len(images_files)} image files\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f39153ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\0.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1008.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1016.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1028.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1060.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1082.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1106.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1119.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1137.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1157.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1175.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1189.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\119.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1201.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1218.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1233.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1246.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1257.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1278.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1312.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\1319.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\141.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\233.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\263.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\37.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\381.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\408.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\437.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\457.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\484.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\508.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\552.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\575.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\590.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\619.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\63.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\647.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\683.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\706.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\721.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\758.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\776.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\805.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\823.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\83.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\836.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\858.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\876.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\893.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\915.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\949.jpg',\n", + " 'FFMPEG_detects\\\\16257fd6-b91b-4d00-a680-9ece9f3f241c\\\\c44f38f6-0186-436f-8c2d-ffb50a539c76\\\\frames\\\\99.jpg']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "images_files" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "40c70986", + "metadata": {}, + "outputs": [], + "source": [ + "# code to create dataset from sampled frames\n", + "\n", + "def upload_images_from_files(images_files, client, client_id):\n", + " \"\"\"Upload specific image files to create a dataset\"\"\"\n", + " \n", + " client.enable_connection_pooling = True\n", + " \n", + " dataset_config = {\n", + " \"client_id\": client_id,\n", + " \"dataset_name\": \"video_sampling_1\",\n", + " \"dataset_description\": \"video sampling dataset from frames\",\n", + " \"data_type\": \"image\", \n", + " }\n", + " \n", + " try:\n", + " response = client.create_dataset(\n", + " dataset_config=dataset_config,\n", + " files_to_upload=images_files \n", + " )\n", + " print(f\"Dataset created successfully!\")\n", + " print(f\"Dataset ID: {response['dataset_id']}\")\n", + " return response['dataset_id']\n", + " except Exception as e:\n", + " print(f\"Error creating dataset: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "a1d96b25", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset created successfully!\n", + "Dataset ID: 6a680901-fe81-49f0-9120-bb754d63a341\n" + ] + }, + { + "data": { + "text/plain": [ + "'6a680901-fe81-49f0-9120-bb754d63a341'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "upload_images_from_files(images_files, client, client_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "958fc75e", + "metadata": {}, + "outputs": [], + "source": [ + "new_dataset_id = '6a680901-fe81-49f0-9120-bb754d63a341'" + ] + }, + { + "cell_type": "markdown", + "id": "b454c4f4", + "metadata": {}, + "source": [ + "### Image Annotation Project Creation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d32106c5", + "metadata": {}, + "outputs": [], + "source": [ + "# modify to add questions to image project\n", + "\n", + "questions = [\n", + " {\n", + " \"question_number\": 1,\n", + " \"question\": \"Test\",\n", + " \"question_id\": \"533bb0c8-fb2b-4394-a8e1-5042a944802f\",\n", + " \"option_type\": \"polygon\",\n", + " \"required\": True,\n", + " \"options\": [\n", + " { \"option_name\": \"#fe1236\" }\n", + " ],\n", + " \"question_metadata\": []\n", + " }\n", + " ]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b71d2aa0", + "metadata": {}, + "outputs": [], + "source": [ + "# creeate the annotation guideline template\n", + "\n", + "template_id = client.create_annotation_guideline(\n", + " client_id=client_id,\n", + " questions=questions,\n", + " template_name=\"video_sampling_template_1\",\n", + " data_type=\"image\",\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83565ec3", + "metadata": {}, + "outputs": [], + "source": [ + "# create the image annotation project\n", + "\n", + "response = client.create_project(\n", + " project_name=\"Video_sampling_project_1\",\n", + " data_type=\"image\",\n", + " client_id= client_id,\n", + " dataset_id=new_dataset_id,\n", + " annotation_template_id=template_id,\n", + " rotation_config={\n", + " \"annotation_rotation_count\": 1,\n", + " \"review_rotation_count\": 1,\n", + " \"client_review_rotation_count\": 1,\n", + " },\n", + " created_by=\"yashsuman15@gmail.com\"\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "9f682f4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Project created successfully!\n", + "Project ID: sherri_puny_rattlesnake_84247\n" + ] + } + ], + "source": [ + "if response['response']['project_id']:\n", + " print(f\"Project created successfully!\")\n", + " print(f\"Project ID: {response['response']['project_id']}\")\n", + " image_project_id = response['response']['project_id']" + ] + }, + { + "cell_type": "markdown", + "id": "8645aa60", + "metadata": {}, + "source": [ + "## 6. Performing Annotations of Image Project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6eea565", + "metadata": {}, + "outputs": [], + "source": [ + "# annotations of image project on labellerr platform" + ] + }, + { + "cell_type": "markdown", + "id": "8f0611f5", + "metadata": {}, + "source": [ + "### Exporting the Annotation Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b78ad296", + "metadata": {}, + "outputs": [], + "source": [ + "# code to export the annotations from image project\n", + "\n", + "export_config = {\n", + " \"export_name\": \"Weekly Export\",\n", + " \"export_description\": \"Export of all accepted annotations\",\n", + " \"export_format\": \"coco_json\",\n", + " \"statuses\": [\n", + " \"review\",\n", + " \"r_assigned\",\n", + " \"client_review\",\n", + " \"cr_assigned\",\n", + " \"accepted\",\n", + " ],\n", + " }\n", + "\n", + "\n", + "response = client.create_local_export(\n", + " project_id=image_project_id,\n", + " client_id=client_id,\n", + " export_config=export_config\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "18529760", + "metadata": {}, + "source": [ + "## 7. Uploading annotations to Video Project" + ] + }, + { + "cell_type": "markdown", + "id": "deae26b8", + "metadata": {}, + "source": [ + "### Trigger SAM2 tracking on Video annotation project\n", + "\n", + "Using the export, retrive the prompt to run SAM2 tracking on video" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "df6b3ac7", + "metadata": {}, + "outputs": [], + "source": [ + "# code to create video annotation project from image annotations export" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "SDk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/labellerr/services/labellerr_files/client_utils.py b/labellerr/services/labellerr_files/client_utils.py new file mode 100644 index 0000000..8a8af1e --- /dev/null +++ b/labellerr/services/labellerr_files/client_utils.py @@ -0,0 +1,424 @@ +from labellerr.client import LabellerrClient +from labellerr.exceptions import LabellerrError +from labellerr import constants +import uuid +import os +import subprocess +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed +from threading import Lock +from abc import ABCMeta, abstractmethod + +class LabellerrFileMeta(ABCMeta): + """Metaclass that combines ABC functionality with factory pattern""" + + def __call__(cls, client, file_id, project_id, dataset_id = None, **kwargs): + + if cls.__name__ != 'LabellerrFile': + + instance = cls.__new__(cls) + if isinstance(instance, cls): + instance.__init__(client, file_id, project_id, dataset_id=dataset_id, **kwargs) + return instance + + + try: + unique_id = str(uuid.uuid4()) + client_id = client.client_id + params = { + 'file_id': file_id, + 'include_answers': 'false', + 'project_id': project_id, + 'uuid': unique_id, + 'client_id': client_id + } + + # TODO: Add dataset_id to params based on precedence logic + # Priority: project_id > dataset_id + + url = f"{constants.BASE_URL}/data/file_data" + response = client.make_api_request(client_id, url, params, unique_id) + + # Extract data_type from response + file_metadata = response.get('file_metadata', {}) + data_type = response.get('data_type', '').lower() + + # print(f"Detected file type: {data_type}") + + # Route to appropriate subclass + if data_type == 'image': + return LabellerrImageFile(client, file_id, project_id, dataset_id=dataset_id, + file_metadata=file_metadata) + elif data_type == 'video': + return LabellerrVideoFile(client, file_id, project_id, dataset_id=dataset_id, + file_metadata=file_metadata) + elif data_type == 'pdf': + return LabellerrPDFFile(client, file_id, project_id, dataset_id=dataset_id, + file_metadata=file_metadata) + else: + raise LabellerrError(f"Unsupported file type: {data_type}") + + + except Exception as e: + raise LabellerrError(f"Failed to create file instance: {str(e)}") + + +class LabellerrFile(metaclass=LabellerrFileMeta): + """Base class for all Labellerr files with factory behavior""" + + def __init__(self, client: LabellerrClient, file_id: str, project_id: str, + dataset_id: str | None = None, **kwargs): + """ + Initialize base file attributes + + :param client: LabellerrClient instance + :param file_id: Unique file identifier + :param project_id: Project ID containing the file + :param dataset_id: Optional dataset ID + :param kwargs: Additional file data (file_metadata, response, etc.) + """ + self.client = client + self.file_id = file_id + self.project_id = project_id + self.client_id = client.client_id + self.dataset_id = dataset_id + + # Store metadata from factory creation + self.metadata = kwargs.get('file_metadata', {}) + + + def get_metadata(self, include_answers: bool = False): + """ + Refresh and retrieve file metadata from Labellerr API. + + :param include_answers: Whether to include annotation answers + :return: Dictionary containing file metadata + """ + try: + unique_id = str(uuid.uuid4()) + + params = { + 'file_id': self.file_id, + 'include_answers': str(include_answers).lower(), + 'project_id': self.project_id, + 'uuid': unique_id, + 'client_id': self.client_id + } + + # TODO: Add dataset_id handling if needed + + url = f"{constants.BASE_URL}/data/file_data" + response = self.client.make_api_request(self.client_id, url, params, unique_id) + + # Update cached metadata + self.metadata = response.get('file_metadata', {}) + + return response + + except Exception as e: + raise LabellerrError(f"Failed to fetch file metadata: {str(e)}") + + +class LabellerrImageFile(LabellerrFile): + pass + +class LabellerrVideoFile(LabellerrFile): + """Specialized class for handling video files including frame operations""" + + def __init__(self, client: LabellerrClient, file_id: str, project_id: str, dataset_id: str | None = None, **kwargs): + super().__init__(client, file_id, project_id, dataset_id=dataset_id, **kwargs) + + @property + def total_frames(self): + """Get total number of frames in the video.""" + return self.metadata.get('total_frames', 0) + + def get_frames(self, frame_start: int = 0, frame_end: int | None = None): + """ + Retrieve video frames data from Labellerr API. + + :param frame_start: Starting frame index (default: 0) + :param frame_end: Ending frame index (default: total_frames) + :return: Dictionary containing video frames data with frame numbers as keys and URLs as values + """ + try: + if self.dataset_id is None: + raise ValueError("dataset_id is required for fetching video frames") + + # Use total_frames as default for frame_end + if frame_end is None: + frame_end = self.total_frames + + unique_id = str(uuid.uuid4()) + url = f"{constants.BASE_URL}/data/video_frames" + + params = { + 'dataset_id': self.dataset_id, + 'file_id': self.file_id, + 'frame_start': frame_start, + 'frame_end': frame_end, + 'project_id': self.project_id, + 'uuid': unique_id, + 'client_id': self.client_id + } + + response = self.client.make_api_request(self.client_id, url, params, unique_id) + + return response + + except Exception as e: + raise LabellerrError(f"Failed to fetch video frames data: {str(e)}") + + def _download_single_frame(self, frame_number, frame_url, save_path, print_lock): + """ + Download a single frame (helper method for threading). + + :param frame_number: Frame number + :param frame_url: URL to download from + :param save_path: Directory to save the frame + :param print_lock: Lock for thread-safe printing + :return: Tuple of (success: bool, frame_number: str, error_info: dict or None) + """ + try: + filename = f"{frame_number}.jpg" + filepath = os.path.join(save_path, filename) + + response = requests.get(frame_url, timeout=30) + + if response.status_code == 200: + with open(filepath, 'wb') as f: + f.write(response.content) + + with print_lock: + print(f"Downloaded: {filename}") + + return True, frame_number, None + else: + error_info = { + 'frame': frame_number, + 'status': response.status_code + } + with print_lock: + print(f"Failed to download frame {frame_number}: Status {response.status_code}") + + return False, frame_number, error_info + + except Exception as e: + error_info = { + 'frame': frame_number, + 'error': str(e) + } + with print_lock: + print(f"Error downloading frame {frame_number}: {str(e)}") + + return False, frame_number, error_info + + def download_frames(self, frames_data: dict, output_folder: str | None = None, + max_workers: int = 10): + """ + Download video frames from URLs to a local folder using multithreading. + + :param frames_data: Dictionary with frame numbers as keys and URLs as values + :param output_folder: Base folder path where frames will be saved (default: current directory) + :param max_workers: Maximum number of concurrent download threads (default: 10) + :return: Dictionary with download statistics + """ + try: + # Use file_id as folder name + folder_name = self.file_id + + # Set output path + if output_folder: + save_path = os.path.join(output_folder, folder_name) + else: + save_path = folder_name + + # Create directory if it doesn't exist + os.makedirs(save_path, exist_ok=True) + + success_count = 0 + failed_frames = [] + print_lock = Lock() + + print(f"Downloading {len(frames_data)} frames to: {save_path}") + print(f"Using {max_workers} concurrent threads") + + # Use ThreadPoolExecutor for concurrent downloads + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # Submit all download tasks + future_to_frame = { + executor.submit( + self._download_single_frame, + frame_number, + frame_url, + save_path, + print_lock + ): frame_number + for frame_number, frame_url in frames_data.items() + } + + # Process completed downloads + for future in as_completed(future_to_frame): + success, frame_number, error_info = future.result() + + if success: + success_count += 1 + else: + failed_frames.append(error_info) + + result = { + 'total_frames': len(frames_data), + 'successful_downloads': success_count, + 'failed_downloads': len(failed_frames), + 'save_path': save_path, + 'failed_frames': failed_frames + } + + # print(f"\nDownload complete: {success_count}/{len(frames_data)} frames downloaded successfully") + + return result + + except Exception as e: + raise LabellerrError(f"Failed to download video frames: {str(e)}") + + def create_video(self, frames_folder: str, output_file: str = "output.mp4", + framerate: int = 30, pattern: str = "%d.jpg"): + """ + Join frames into a video using ffmpeg. + + :param frames_folder: Path to folder containing sequential frames (e.g., 1.jpg, 2.jpg). + :param output_file: Name of the output video file (default: output.mp4). + :param framerate: Desired video framerate (default: 30 fps). + :param pattern: Pattern for sequential frames (default: %d.jpg → 1.jpg, 2.jpg, ...). + :return: Path to created video file + """ + if frames_folder is None: + raise ValueError("frames_folder must be provided") + + input_pattern = os.path.join(frames_folder, pattern) + + # FFmpeg command + command = [ + "ffmpeg", + "-y", # Overwrite output file if exists + "-framerate", str(framerate), + "-i", input_pattern, + "-c:v", "libx264", + "-pix_fmt", "yuv420p", + output_file + ] + + try: + print("Running command:", " ".join(command)) + subprocess.run(command, check=True) + print(f"Video saved as {output_file}") + return output_file + except subprocess.CalledProcessError as e: + raise LabellerrError(f"Error while joining frames: {str(e)}") + +class LabellerrVideoDataset: + """ + Class for handling video dataset operations and fetching multiple video files. + """ + + def __init__(self, client: LabellerrClient, dataset_id: str, project_id: str): + """ + Initialize video dataset instance. + + :param client: LabellerrClient instance + :param dataset_id: Dataset ID + :param project_id: Project ID containing the dataset + """ + self.client = client + self.dataset_id = dataset_id + self.project_id = project_id + self.client_id = client.client_id + + def fetch_files(self, limit: int | None = None, page_size: int = 10): + """ + Fetch all video files in this dataset as LabellerrVideoFile instances. + + :param limit: Maximum number of files to fetch (None for all) + :param page_size: Number of files to fetch per API request (default: 10) + :return: List of LabellerrVideoFile instances + """ + try: + all_file_ids = [] + next_search_after = "" # Start with empty string for first page + + # while True: + unique_id = str(uuid.uuid4()) + url = f"{constants.BASE_URL}/search/files/all" + params = { + 'sort_by': 'created_at', + 'sort_order': 'desc', + 'size': page_size, + 'next_search_after': next_search_after, + 'uuid': unique_id, + 'dataset_id': self.dataset_id, + 'client_id': self.client_id + } + + response = self.client.make_api_request(self.client_id, url, params, unique_id) + print(response) + + + # Create LabellerrVideoFile instances for each file_id + # video_files = [] + # print(f"\nCreating LabellerrFile instances for {len(all_file_ids)} files...") + + # for file_id in all_file_ids: + # try: + # video_file = LabellerrFile( + # client=self.client, + # file_id=file_id, + # project_id=self.project_id, + # dataset_id=self.dataset_id + # ) + # video_files.append(video_file) + # except Exception as e: + # print(f"Warning: Failed to create file instance for {file_id}: {str(e)}") + + except Exception as e: + raise LabellerrError(f"Failed to fetch dataset files: {str(e)}") + +# Example usage +if __name__ == "__main__": + + api_key = "66f4d8.9f402742f58a89568f5bcc0f86" + api_secret = "1e2478b930d4a842a526beb585e60d2a9ee6a6f1e3aa89cb3c8ead751f418215" + client_id = "14078" + dataset_id = "16257fd6-b91b-4d00-a680-9ece9f3f241c" + project_id = "gabrila_artificial_duck_74237" + file_id = "c44f38f6-0186-436f-8c2d-ffb50a539c76" + + client = LabellerrClient(api_key=api_key, api_secret=api_secret, client_id=client_id) + + lb_file = LabellerrFile( + client=client, + file_id=file_id, + project_id=project_id, + dataset_id=dataset_id + ) + + + # print(f"File type: {type(lb_file).__name__}") + + # if isinstance(lb_file, LabellerrVideoFile): + # print(f"Total frames: {lb_file.total_frames}") + + # Get video frames + # frames = lb_file.get_frames() + + # Download frames + # lb_file.download_frames(frames, output_folder="./output") + + # Create video from frames + # frames_path = f"./output/{file_id}" + # lb_file.create_video(frames_folder=frames_path, output_file="final_video.mp4", framerate=30) + + lb_dataset = LabellerrVideoDataset(client=client, dataset_id=dataset_id, project_id=project_id) + lb_dataset.fetch_files() + # print(f"Fetched {len(video_files)} video files from dataset {dataset_id}") + # print(video_files) + \ No newline at end of file diff --git a/labellerr/services/video_sampling/__init__.py b/labellerr/services/video_sampling/__init__.py index 2fb5def..d31a892 100644 --- a/labellerr/services/video_sampling/__init__.py +++ b/labellerr/services/video_sampling/__init__.py @@ -2,3 +2,12 @@ All algorithms for video sampling will go in separate files. """ +from .ffmpeg import FFMPEGSceneDetect +from .pyscene_detect import PySceneDetect +from .ssim import SSIMSceneDetect + +__all__ = [ + 'FFMPEGSceneDetect', + 'PySceneDetect', + 'SSIMSceneDetect', +] diff --git a/labellerr/services/video_sampling/ffmpeg.py b/labellerr/services/video_sampling/ffmpeg.py new file mode 100644 index 0000000..6f72c42 --- /dev/null +++ b/labellerr/services/video_sampling/ffmpeg.py @@ -0,0 +1,151 @@ +import subprocess +import os +import json +from pydantic import BaseModel, Field +from typing import List +from labellerr.base.singleton import Singleton + + +class SceneFrame(BaseModel): + """Represents an extracted keyframe.""" + frame_path: str + frame_index: int + + +class DetectionResult(BaseModel): + """Contains all extraction results for a video.""" + file_id: str + output_folder: str + selected_frames: List[SceneFrame] = Field(default_factory=list) + + +class FFMPEGSceneDetect(Singleton): + """Keyframe extraction from videos using FFMPEG (Singleton).""" + + def detect_and_extract(self, video_path: str) -> DetectionResult: + """ + Extract keyframes from video and save to detects folder structure. + + Args: + video_path: Path to the video file + + Returns: + DetectionResult containing file_id, output_folder, and list of SceneFrame objects + """ + # Derive file_id from video_path (base name without extension) + file_id = os.path.splitext(os.path.basename(video_path))[0] + dataset_id = os.path.basename(os.path.dirname(video_path)) + + # Create detects folder structure + base_detect_folder = "FFMPEG_detects" + + output_folder = os.path.join(base_detect_folder, dataset_id, file_id) + frames_folder = os.path.join(output_folder, "frames") + + # Create nested folders + os.makedirs(frames_folder, exist_ok=True) + + # Update output pattern to use frames subfolder in detects structure + output_pattern = os.path.join(frames_folder, "%d.jpg") + + command = [ + "ffmpeg", + "-i", video_path, + "-vf", "select='eq(pict_type,PICT_TYPE_I)',showinfo", + "-vsync", "vfr", + "-frame_pts", "1", + output_pattern + ] + + try: + result = subprocess.run(command, check=True, capture_output=True, text=True) + print(f"Keyframes extracted to {frames_folder}") + + # Parse frame information from FFMPEG output + selected_frames = self._parse_ffmpeg_output(result.stderr, frames_folder) + + # Create result + detection_result = DetectionResult( + file_id=file_id, + output_folder=output_folder, # Main detects/file_id folder + selected_frames=selected_frames + ) + + # Save JSON mapping + self._save_json_mapping(detection_result, output_folder, file_id) + + return detection_result + + except subprocess.CalledProcessError as e: + print(f"Error extracting keyframes: {e}") + raise + + def _parse_ffmpeg_output(self, stderr_output: str, frames_folder: str) -> List[SceneFrame]: + """ + Parse FFMPEG stderr output to extract frame information. + + Args: + stderr_output: FFMPEG stderr output containing showinfo data + frames_folder: Folder where frames are saved (detects/file_id/frames) + + Returns: + List of SceneFrame objects + """ + frames = [] + frame_counter = 1 + + # Parse showinfo output from stderr + for line in stderr_output.split('\n'): + if 'showinfo' in line and 'n:' in line: + # The frame file is named sequentially starting from 1 + frame_path = os.path.join(frames_folder, f"{frame_counter}.jpg") + + # Extract frame number from showinfo line if needed + # Example: [Parsed_showinfo_1 @ 0x...] n: 0 pts: 0 ... + try: + if 'pts_time:' in line: + # Extract the actual frame number from the source + parts = line.split('n:') + if len(parts) > 1: + frame_no = int(parts[1].split()[0]) + else: + frame_no = frame_counter - 1 + else: + frame_no = frame_counter - 1 + + frames.append(SceneFrame( + frame_path=frame_path, + frame_index=frame_no + )) + frame_counter += 1 + except (ValueError, IndexError): + continue + + return frames + + def _save_json_mapping(self, result: DetectionResult, output_folder: str, file_id: str) -> None: + """ + Save JSON mapping of file_id to extracted keyframes. + + Args: + result: DetectionResult object + output_folder: Folder to save the JSON file (detects/file_id/) + file_id: Unique identifier for the video + """ + # Use Pydantic's model_dump + result_dict = result.model_dump() + result_dict["total_selected_frames"] = len(result.selected_frames) + + json_path = os.path.join(output_folder, f"{file_id}_mapping.json") + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(result_dict, f, indent=2, ensure_ascii=False) + + print(f"JSON mapping saved to: {json_path}") + + +if __name__ == "__main__": + video_path = r"D:\professional\LABELLERR\Task\Repos\SDKPython\download_video\59438ec3-12e0-4687-8847-1e6e01b0bf25\1cb2eec4-5125-4272-ad09-c249f40fffb3.mp4" + + # Get singleton instance + detector = FFMPEGSceneDetect() + result = detector.detect_and_extract(video_path) \ No newline at end of file diff --git a/labellerr/services/video_sampling/ffmpeg_sampling.py b/labellerr/services/video_sampling/ffmpeg_sampling.py deleted file mode 100644 index e69de29..0000000 diff --git a/labellerr/services/video_sampling/gemini.py b/labellerr/services/video_sampling/gemini.py new file mode 100644 index 0000000..16a4d65 --- /dev/null +++ b/labellerr/services/video_sampling/gemini.py @@ -0,0 +1,263 @@ +import os +import cv2 +from PIL import Image +from pydantic import BaseModel, Field +from typing import List, Optional +import json +from google.cloud import videointelligence +from labellerr.base.singleton import Singleton + + +class SceneFrame(BaseModel): + """Represents a detected scene with its extracted frame.""" + frame_path: str + frame_no: int + start_time_offset: float + end_time_offset: float + + +class DetectionResult(BaseModel): + """Contains all detection results for a video.""" + file_id: str + output_folder: str + total_frames: int + selected_frames: List[SceneFrame] = Field(default_factory=list) + + +class GeminiSceneDetect(Singleton): + """Google Cloud Video Intelligence API scene detection and frame extraction.""" + + def detect_and_extract( + self, + video_path: str, + file_id: str, + gcs_uri: Optional[str] = None, + credentials_path: Optional[str] = None + ) -> DetectionResult: + """ + Detect scenes using Google Cloud Video Intelligence API and extract representative frames. + + Args: + video_path: Path to the local video file (for frame extraction) + file_id: Unique identifier for the video (used as output folder name) + gcs_uri: Google Cloud Storage URI (gs://bucket/video.mp4) for API processing. + If None, the video will be uploaded as bytes (limited to 10MB) + credentials_path: Path to service account JSON key file. + If None, uses GOOGLE_APPLICATION_CREDENTIALS environment variable + + Returns: + DetectionResult containing file_id, output_folder, total_frames, and list of SceneFrame objects + """ + output_folder = file_id + + # Set credentials if provided + if credentials_path: + os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_path + + # Initialize Video Intelligence client + client = videointelligence.VideoIntelligenceServiceClient() + + print(f"Processing video: {video_path}") + print("Detecting shot changes using Google Cloud Video Intelligence API...") + + # Detect shots using Video Intelligence API + shots = self._detect_shots(client, video_path, gcs_uri) + + if not shots: + raise ValueError("No shot changes detected in the video") + + print(f"Detected {len(shots)} shots") + + # Create output folder + os.makedirs(output_folder, exist_ok=True) + + # Open video for frame extraction + video = cv2.VideoCapture(video_path) + + if not video.isOpened(): + raise ValueError(f"Cannot open video: {video_path}") + + # Get video properties + total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = video.get(cv2.CAP_PROP_FPS) + + print(f"Total frames: {total_frames}") + print(f"FPS: {fps}") + + # Extract and save frames + scene_frames = [] + + for idx, shot in enumerate(shots): + # Calculate middle frame number from shot timestamps + start_time = shot.start_time_offset.total_seconds() + end_time = shot.end_time_offset.total_seconds() + middle_time = (start_time + end_time) / 2 + frame_no = int(middle_time * fps) + + # Ensure frame number is within bounds + frame_no = max(0, min(frame_no, total_frames - 1)) + + # Extract frame + frame = self._get_frame(video, frame_no) + + if frame is None: + print(f"Warning: Could not extract frame {frame_no} for shot {idx}") + continue + + # Save frame with frame number as filename + frame_filename = f"{frame_no}.jpg" + frame_path = os.path.join(output_folder, frame_filename) + frame.save(frame_path) + + # Create SceneFrame object + scene_frame = SceneFrame( + frame_path=frame_path, + frame_no=frame_no, + start_time_offset=start_time, + end_time_offset=end_time + ) + scene_frames.append(scene_frame) + + print(f"Saved keyframe {idx} at frame {frame_no} (time: {middle_time:.2f}s)") + + video.release() + + print(f"\nExtracted {len(scene_frames)} keyframes from {total_frames} frames.") + + # Create result + result = DetectionResult( + file_id=file_id, + output_folder=output_folder, + total_frames=total_frames, + selected_frames=scene_frames + ) + + # Save JSON mapping + self._save_json_mapping(result, output_folder, file_id, gcs_uri) + + return result + + def _detect_shots( + self, + client: videointelligence.VideoIntelligenceServiceClient, + video_path: str, + gcs_uri: Optional[str] + ) -> List: + """ + Detect shot changes using Google Cloud Video Intelligence API. + + Args: + client: Video Intelligence client instance + video_path: Path to the local video file + gcs_uri: Google Cloud Storage URI + + Returns: + List of shot annotation objects + """ + features = [videointelligence.Feature.SHOT_CHANGE_DETECTION] + + if gcs_uri: + # Use GCS URI for large videos + print(f"Analyzing video from GCS: {gcs_uri}") + operation = client.annotate_video( + request={ + "input_uri": gcs_uri, + "features": features + } + ) + else: + # Read video file and send as bytes (limited to 10MB) + with open(video_path, "rb") as video_file: + input_content = video_file.read() + + print(f"Analyzing video from local file (size: {len(input_content) / (1024*1024):.2f} MB)") + + if len(input_content) > 10 * 1024 * 1024: # 10MB limit + raise ValueError( + "Video file is larger than 10MB. Please upload to Google Cloud Storage " + "and provide gcs_uri parameter (gs://bucket/video.mp4)" + ) + + operation = client.annotate_video( + request={ + "input_content": input_content, + "features": features + } + ) + + print("Waiting for operation to complete...") + result = operation.result(timeout=600) # 10 minute timeout + + # Get shot annotations + annotation_result = result.annotation_results[0] + shots = annotation_result.shot_annotations + + return shots + + def _get_frame(self, video: cv2.VideoCapture, frame_no: int) -> Optional[Image.Image]: + """ + Extract a specific frame from video. + + Args: + video: OpenCV video capture object + frame_no: Frame number to extract + + Returns: + PIL Image of the frame, or None if extraction fails + """ + video.set(cv2.CAP_PROP_POS_FRAMES, frame_no) + success, frame = video.read() + + if not success: + return None + + return Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + + def _save_json_mapping( + self, + result: DetectionResult, + output_folder: str, + file_id: str, + gcs_uri: Optional[str] + ) -> None: + """ + Save JSON mapping of file_id to extracted scenes. + + Args: + result: DetectionResult object + output_folder: Folder to save the JSON file + file_id: Unique identifier for the video + gcs_uri: Google Cloud Storage URI (if used) + """ + # Use Pydantic's model_dump + result_dict = result.model_dump() + result_dict["total_selected_frames"] = len(result.selected_frames) + result_dict["gcs_uri"] = gcs_uri if gcs_uri else "local file" + + json_path = os.path.join(output_folder, f"{file_id}_mapping.json") + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(result_dict, f, indent=2, ensure_ascii=False) + + print(f"JSON mapping saved to: {json_path}") + + +if __name__ == "__main__": + # Example usage - Local video file (must be < 10MB) + video_path = r"D:\professional\LABELLERR\Task\Repos\SDKPython\labellerr\Python_SDK\services\video_sampling\video2.mp4" + cred_json_path = r"D:\professional\LABELLERR\Task\Repos\SDKPython\labellerr\Python_SDK\services\video_sampling\yash-suman-prod.json" + + # Get singleton instance + detector = GeminiSceneDetect() + + # Detect and extract frames + try: + result = detector.detect_and_extract( + video_path=video_path, + file_id="video_001", + gcs_uri=None, # Set to gs://bucket/video.mp4 for large videos + credentials_path=cred_json_path + ) + + + except Exception as e: + print(f"Error: {e}") \ No newline at end of file diff --git a/labellerr/services/video_sampling/gemini_sampling.py b/labellerr/services/video_sampling/gemini_sampling.py deleted file mode 100644 index e69de29..0000000 diff --git a/labellerr/services/video_sampling/pyscene_detect.py b/labellerr/services/video_sampling/pyscene_detect.py new file mode 100644 index 0000000..d430283 --- /dev/null +++ b/labellerr/services/video_sampling/pyscene_detect.py @@ -0,0 +1,135 @@ +import os +from scenedetect import detect, AdaptiveDetector +from PIL import Image +import cv2 +from pydantic import BaseModel, Field +from typing import List +import json +from labellerr.base.singleton import Singleton + + +class SceneFrame(BaseModel): + """Represents a detected scene with its extracted frame.""" + frame_path: str + frame_index: int + + +class DetectionResult(BaseModel): + """Contains all detection results for a video.""" + file_id: str + output_folder: str + total_frames: int + selected_frames: List[SceneFrame] = Field(default_factory=list) + + +class PySceneDetect(Singleton): + """Scene detection and frame extraction for videos (Singleton).""" + + def detect_and_extract(self, video_path: str) -> DetectionResult: + """ + Detect scenes and extract representative frames. + + Args: + video_path: Path to the video file + + Returns: + DetectionResult containing file_id, output_folder, total_frames, and list of SceneFrame objects + """ + # Derive file_id from video_path (base name without extension) + file_id = os.path.splitext(os.path.basename(video_path))[0] + dataset_id = os.path.basename(os.path.dirname(video_path)) + + # Create base detect folder and file_id specific folder + base_detect_folder = "PyScene_detects" + + output_folder = os.path.join(base_detect_folder, dataset_id, file_id) + frames_folder = os.path.join(output_folder, "frames") # New frames subfolder + + # Detect scene transitions + scenes = detect(video_path, AdaptiveDetector()) + + # Create nested output folders + os.makedirs(frames_folder, exist_ok=True) # Create frames subfolder + + # Open video for frame extraction + video = cv2.VideoCapture(video_path) + + # Get total frames in video + total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + + # Extract and save frames + scene_frames = [] + for scene in scenes: + # Calculate middle frame number + frame_no = (scene[1] - scene[0]).frame_num // 2 + scene[0].frame_num + + # Extract frame + frame = self._get_frame(video, frame_no) + + # Save frame with frame number as filename inside frames folder + frame_filename = f"{frame_no}.jpg" + frame_path = os.path.join(frames_folder, frame_filename) # Updated path + frame.save(frame_path) + + # Create SceneFrame object + scene_frame = SceneFrame( + frame_path=frame_path, + frame_index=frame_no + ) + scene_frames.append(scene_frame) + + video.release() + + # Create result + result = DetectionResult( + file_id=file_id, + output_folder=output_folder, + total_frames=total_frames, + selected_frames=scene_frames + ) + + # Save JSON mapping + self._save_json_mapping(result, output_folder, file_id) + + return result + + def _get_frame(self, video: cv2.VideoCapture, frame_no: int) -> Image.Image: + """ + Extract a specific frame from video. + + Args: + video: OpenCV video capture object + frame_no: Frame number to extract + + Returns: + PIL Image of the frame + """ + video.set(cv2.CAP_PROP_POS_FRAMES, frame_no) + _, frame = video.read() + return Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + + def _save_json_mapping(self, result: DetectionResult, output_folder: str, file_id: str) -> None: + """ + Save JSON mapping of file_id to extracted scenes. + + Args: + result: DetectionResult object + output_folder: Folder to save the JSON file + file_id: Unique identifier for the video + """ + # Use Pydantic's model_dump instead of asdict + result_dict = result.model_dump() + result_dict["total_selected_frames"] = len(result.selected_frames) + + json_path = os.path.join(output_folder, f"{file_id}_mapping.json") + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(result_dict, f, indent=2, ensure_ascii=False) + + print(f"JSON mapping saved to: {json_path}") + + +# if __name__ == "__main__": +# video_path = r"D:\professional\LABELLERR\Task\Repos\SDKPython\labellerr\notebooks\c44f38f6-0186-436f-8c2d-ffb50a539c76.mp4" + +# detector = PySceneDetect() +# result = detector.detect_and_extract(video_path) \ No newline at end of file diff --git a/labellerr/services/video_sampling/requirements.txt b/labellerr/services/video_sampling/requirements.txt new file mode 100644 index 0000000..aa716f2 --- /dev/null +++ b/labellerr/services/video_sampling/requirements.txt @@ -0,0 +1,3 @@ +scenedetect +google-cloud-videointelligence +scikit-image \ No newline at end of file diff --git a/labellerr/services/video_sampling/ssim.py b/labellerr/services/video_sampling/ssim.py new file mode 100644 index 0000000..0e1f9ba --- /dev/null +++ b/labellerr/services/video_sampling/ssim.py @@ -0,0 +1,225 @@ +import os +import cv2 +import numpy as np +from PIL import Image +from pydantic import BaseModel, Field +from typing import List +import json +from skimage.metrics import structural_similarity as ssim +from labellerr.base.singleton import Singleton + + +class SceneFrame(BaseModel): + """Represents a detected scene with its extracted frame.""" + frame_path: str + frame_index: int + ssim_score: float + + +class DetectionResult(BaseModel): + """Contains all detection results for a video.""" + file_id: str + output_folder: str + total_frames: int + selected_frames: List[SceneFrame] = Field(default_factory=list) + + +class SSIMSceneDetect(Singleton): + """SSIM-based scene detection and frame extraction for videos (Singleton).""" + + def detect_and_extract( + self, + video_path: str, + threshold: float = 0.6, + resize_dim: tuple = (320, 240) + ) -> DetectionResult: + """ + Detect scenes using SSIM and extract representative frames. + + Args: + video_path: Path to the video file + threshold: SSIM threshold for scene detection (lower = stricter, default: 0.6) + resize_dim: Dimensions to resize frames for SSIM calculation (default: (320, 240)) + + Returns: + DetectionResult containing file_id, output_folder, total_frames, and list of SceneFrame objects + """ + # Derive file_id from video_path (base name without extension) + file_id = os.path.splitext(os.path.basename(video_path))[0] + dataset_id = os.path.basename(os.path.dirname(video_path)) + + # Create detects folder structure + base_detect_folder = "SSIM_detects" + output_folder = os.path.join(base_detect_folder, dataset_id, file_id) + frames_folder = os.path.join(output_folder, "frames") + + # Create nested output folders + os.makedirs(frames_folder, exist_ok=True) + + # Open video for processing + video = cv2.VideoCapture(video_path) + + if not video.isOpened(): + raise ValueError(f"Cannot open video: {video_path}") + + # Get total frames in video + total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + + print(f"Processing video: {video_path}") + print(f"Total frames: {total_frames}") + print(f"SSIM threshold: {threshold}") + + # Read first frame + success, prev_frame = video.read() + if not success: + video.release() + raise ValueError(f"Cannot read first frame from: {video_path}") + + # Extract and save frames + scene_frames = [] + frame_count = 0 + + # Always save first frame + self._save_frame(prev_frame, frame_count, 1.0, scene_frames, frames_folder) + # print(f"Saved keyframe 0 at frame {frame_count} (First frame)") + + # Process remaining frames + while True: + success, curr_frame = video.read() + if not success: + break + + frame_count += 1 + + # Calculate SSIM between current and previous frame + ssim_score = self._calculate_ssim(prev_frame, curr_frame, resize_dim) + + # If SSIM is below threshold, it's a scene change + if ssim_score < threshold: + self._save_frame(curr_frame, frame_count, ssim_score, scene_frames, frames_folder) + print(f"Saved keyframe {len(scene_frames) - 1} at frame {frame_count} (SSIM: {ssim_score:.3f})") + prev_frame = curr_frame + elif frame_count % 100 == 0: + print(f"Frame {frame_count}: SSIM = {ssim_score:.3f} (threshold: {threshold})") + + video.release() + + # print(f"\nExtracted {len(scene_frames)} keyframes from {frame_count + 1} frames.") + + # Create result + result = DetectionResult( + file_id=file_id, + output_folder=output_folder, # Main detects/file_id folder + total_frames=total_frames, + selected_frames=scene_frames + ) + + # Save JSON mapping + self._save_json_mapping(result, output_folder, file_id, threshold, resize_dim) + + return result + + def _calculate_ssim(self, frame1: np.ndarray, frame2: np.ndarray, resize_dim: tuple) -> float: + """ + Calculate SSIM score between two frames. + + Args: + frame1: First frame (BGR format) + frame2: Second frame (BGR format) + resize_dim: Dimensions to resize frames for SSIM calculation + + Returns: + SSIM score (0-1, where 1 is identical) + """ + # Resize frames for faster computation + gray1 = cv2.cvtColor(cv2.resize(frame1, resize_dim), cv2.COLOR_BGR2GRAY) + gray2 = cv2.cvtColor(cv2.resize(frame2, resize_dim), cv2.COLOR_BGR2GRAY) + + # Calculate SSIM + score, _ = ssim(gray1, gray2, full=True) + + return score + + def _save_frame( + self, + frame: np.ndarray, + frame_no: int, + ssim_score: float, + scene_frames: List[SceneFrame], + frames_folder: str + ) -> None: + """ + Save a frame to disk and add to scene_frames list. + + Args: + frame: Frame to save (BGR format) + frame_no: Frame number + ssim_score: SSIM score that triggered this frame + scene_frames: List to append SceneFrame object to + frames_folder: Folder to save the frame (detects/file_id/frames) + """ + # Convert BGR to RGB for PIL + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_image = Image.fromarray(frame_rgb) + + # Save frame with frame number as filename in frames folder + frame_filename = f"{frame_no}.jpg" + frame_path = os.path.join(frames_folder, frame_filename) # Now uses frames_folder + pil_image.save(frame_path) + + # Create SceneFrame object + scene_frame = SceneFrame( + frame_path=frame_path, + frame_index=frame_no, + ssim_score=ssim_score + ) + scene_frames.append(scene_frame) + + def _save_json_mapping( + self, + result: DetectionResult, + output_folder: str, # This is now detects/file_id/ + file_id: str, + threshold: float, + resize_dim: tuple + ) -> None: + """ + Save JSON mapping of file_id to extracted scenes. + + Args: + result: DetectionResult object + output_folder: Folder to save the JSON file (detects/file_id/) + file_id: Unique identifier for the video + threshold: SSIM threshold used + resize_dim: Resize dimensions used + """ + # Use Pydantic's model_dump + result_dict = result.model_dump() + result_dict["total_selected_frames"] = len(result.selected_frames) + result_dict["threshold"] = threshold + result_dict["resize_dim"] = resize_dim + + json_path = os.path.join(output_folder, f"{file_id}_mapping.json") + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(result_dict, f, indent=2, ensure_ascii=False) + + print(f"JSON mapping saved to: {json_path}") + + +if __name__ == "__main__": + # Example usage + video_path = r"D:\professional\LABELLERR\Task\Repos\Python_SDK\services\video_sampling\video2.mp4" + + # Get singleton instance + detector = SSIMSceneDetect() + + # Detect and extract frames + result = detector.detect_and_extract( + video_path=video_path, + threshold=0.6, # Lower value = more sensitive to changes + resize_dim=(320, 240) + ) + + print(f"\nDetection complete!") + print(f"Total frames extracted: {len(result.selected_frames)}") + print(f"Output folder: {result.output_folder}") \ No newline at end of file