Feature/labimp 7661 create gcs connector

.bumpversion.cfg

@@ -7,4 +7,4 @@ message = [RELEASE] Bump version: {current_version} → {new_version}
 
 [bumpversion:file:pyproject.toml]
 search = version = "{current_version}"
-replace = version = "{new_version}"
+replace = version = "{new_version}"

.env.example

@@ -0,0 +1,21 @@
+# Labellerr SDK Integration Test Environment Variables
+# Copy this file to .env and update with your actual credentials
+
+# API Credentials
+API_KEY=your_api_key_here
+API_SECRET=your_api_secret_here
+CLIENT_ID=your_client_id_here
+
+# Test Email
+CLIENT_EMAIL=your_test_email@example.com
+
+# AWS Connection Credentials (JSON format)
+# For video data type
+AWS_CONNECTION_VIDEO={"access_key": "your_aws_access_key", "secret_key": "your_aws_secret_key", "s3_path": "your_s3_bucket_path", "data_type": "video", "name": "Video Connection", "description": "AWS S3 connection for video data"}
+
+# For image data type
+AWS_CONNECTION_IMAGE={"access_key": "your_aws_access_key", "secret_key": "your_aws_secret_key", "s3_path": "your_s3_bucket_path", "data_type": "image", "name": "Image Connection", "description": "AWS S3 connection for image data"}
+
+# Optional: Additional test configuration
+# TEST_TIMEOUT=300
+# DEBUG_MODE=false

.flake8

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 160
 extend-ignore = E203, W503, E402
-exclude = .git,__pycache__,.venv,build,dist,venv
+exclude = .git,__pycache__,.venv,build,dist,venv

.github/workflows/ci.yml

@@ -1,71 +1,59 @@
-name: CI Pipeline
+name: CI
 
 on:
   push:
     branches: [ main, develop ]
   pull_request:
     branches: [ main, develop ]
 
-env:
-  PYTHON_VERSION: '3.9'
-
 jobs:
   test:
-    name: Test Suite
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.9']
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e ".[dev]"
-
-    - name: Run linting
-      run: |
-        make lint
-    - name: Run linting
-      run: |
-        make format 
-    - name: Run tests
-      run: |
-        make test
 
-  integration-test:
-    name: Integration Tests
-    runs-on: ubuntu-latest
-    needs: test
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
-
+    env:
+      API_KEY: ${{ secrets.API_KEY }}
+      API_SECRET: ${{ secrets.API_SECRET }}
+      CLIENT_ID: ${{ secrets.CLIENT_ID }}
+      TEST_EMAIL: ${{ secrets.TEST_EMAIL }}
+      AWS_CONNECTION_IMAGE: ${{ secrets.AWS_CONNECTION_IMAGE }}
+      AWS_CONNECTION_VIDEO: ${{ secrets.AWS_CONNECTION_VIDEO }}
+      GCS_CONNECTION_IMAGE: ${{ secrets.GCS_CONNECTION_IMAGE }}
+      GCS_CONNECTION_VIDEO: ${{ secrets.GCS_CONNECTION_VIDEO }}
+
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.9'
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e ".[dev]"
-
-    - name: Run integration tests
-      env:
-        LABELLERR_API_KEY: ${{ secrets.LABELLERR_API_KEY }}
-        LABELLERR_API_SECRET: ${{ secrets.LABELLERR_API_SECRET }}
-        LABELLERR_CLIENT_ID: ${{ secrets.LABELLERR_CLIENT_ID }}
-        LABELLERR_TEST_EMAIL: ${{ secrets.LABELLERR_TEST_EMAIL }}
-      run: |
-        python -m pytest labellerr_use_case_tests.py -v
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run linting
+        run: |
+          if grep -q "^lint:" Makefile; then
+            make lint
+          else
+            pip install ruff
+            ruff check .
+          fi
+
+      - name: Run formatting check
+        run: |
+          if grep -q "^format:" Makefile; then
+            make format
+          else
+            pip install black
+            black --check .
+          fi
+
+      - name: Run unit tests
+        run: make test
+
+      - name: Run integration tests
+        run: make integration-test

.github/workflows/release.yml

@@ -88,8 +88,12 @@ jobs:
           LABELLERR_API_SECRET: ${{ secrets.LABELLERR_API_SECRET }}
           LABELLERR_CLIENT_ID: ${{ secrets.LABELLERR_CLIENT_ID }}
           LABELLERR_TEST_EMAIL: ${{ secrets.LABELLERR_TEST_EMAIL }}
+          AWS_CONNECTION_IMAGE: ${{ secrets.AWS_CONNECTION_IMAGE }}
+          AWS_CONNECTION_VIDEO: ${{ secrets.AWS_CONNECTION_VIDEO }}
+          GCS_CONNECTION_IMAGE: ${{ secrets.GCS_CONNECTION_IMAGE }}
+          GCS_CONNECTION_VIDEO: ${{ secrets.GCS_CONNECTION_VIDEO }}
         run: |
-          python -m pytest labellerr_use_case_tests.py -v
+          python -m pytest labellerr_integration_case_tests.py -v
 
   release:
     name: Create Release
@@ -256,4 +260,4 @@ jobs:
           echo "❌ Release failed!"
           echo "Release job: ${{ needs.release.result }}"
           echo "Build job: ${{ needs.build.result }}"
-          exit 1
+          exit 1

.gitignore

@@ -19,53 +19,10 @@ sdist/
 var/
 wheels/
 *.egg-info/
-.installed.cfg
-*.egg
-
-# IDE
+*.pyc
+*/*.pyc
 .idea
-.vscode/
-*.swp
-*.swo
-*~
-
-# OS
-.DS_Store
-.DS_Store?
-._*
-.Spotlight-V100
-.Trashes
-ehthumbs.db
-Thumbs.db
-
-# Environment
 .env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-# Testing
-.coverage
-htmlcov/
-.pytest_cache/
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.tox/
-
-# Documentation
-docs/_build/
-site/
-
-# Release files
-.bumpversion.cfg.bak
-*.bak
-
-# Claude
+.DS_Store
 .claude
-
-# Test data
-tests/test_data
+tests/test_data

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: debug-statements
+      - id: mixed-line-ending
+
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+        args: ["--line-length=88"]
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: ["--profile=black", "--line-length=88"]
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: []
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
+    hooks:
+      - id: mypy
+        args: ["--config=pyproject.toml"]
+        additional_dependencies:
+          - types-requests
+          - types-aiofiles
+        files: ^labellerr/
+
+ci:
+  autoupdate_schedule: quarterly
+  skip: []

Makefile

@@ -34,7 +34,7 @@ format:
 build: ## Build package
 	$(PYTHON) -m build
 
-version: 
+version:
 	@grep '^version = ' pyproject.toml | cut -d'"' -f2 | sed 's/^/Current version: /' || echo "Version not found"
 
 info:
@@ -54,4 +54,11 @@ check-release: ## Check if everything is ready for release
 	@echo "1. Create feature branch: git checkout -b feature/LABIMP-XXXX-release-vX.X.X"
 	@echo "2. Update version in pyproject.toml"
 	@echo "3. Commit: git commit -m '[LABIMP-XXXX] Prepare release vX.X.X'"
-	@echo "4. Push and create PR to main (patch) or develop (minor)"
+	@echo "4. Push and create PR to main (patch) or develop (minor)"
+
+integration-test:
+	$(PYTHON) -m pytest  -v labellerr_integration_case_tests.py
+
+pre-commit-install:
+	pip install pre-commit
+	pre-commit install