From cec2e041efd9f379e37de36f98ab1d7a8c9d4038 Mon Sep 17 00:00:00 2001 From: Gaurav <> Date: Wed, 16 Jul 2025 13:34:37 +0530 Subject: [PATCH 1/6] make sdk modular --- .flake8 | 4 + .github/workflows/ci.yml | 42 + .github/workflows/claude-code-review.yml | 73 + .gitignore | 6 +- Makefile | 44 + labellerr/__init__.py | 25 +- .../__pycache__/__init__.cpython-310.pyc | Bin 164 -> 0 bytes labellerr/__pycache__/client.cpython-310.pyc | Bin 29572 -> 0 bytes .../__pycache__/exceptions.cpython-310.pyc | Bin 375 -> 0 bytes labellerr/async_client.py | 328 ++++ labellerr/base/singleton.py | 31 +- labellerr/client.py | 1407 +++++++++-------- labellerr/config.py | 2 +- labellerr/constants.py | 44 +- labellerr/exceptions.py | 3 +- labellerr/gcs.py | 105 +- labellerr/utils.py | 36 +- pyproject.toml | 127 ++ requirements.txt | 2 - setup.py | 24 - tests/__pycache__/__init__.cpython-310.pyc | Bin 155 -> 0 bytes tests/__pycache__/test_client.cpython-310.pyc | Bin 1542 -> 0 bytes tests/__pycache__/test_client.cpython-312.pyc | Bin 2626 -> 0 bytes tests/test_client.py | 334 ++-- 24 files changed, 1775 insertions(+), 862 deletions(-) create mode 100644 .flake8 create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/claude-code-review.yml create mode 100644 Makefile delete mode 100644 labellerr/__pycache__/__init__.cpython-310.pyc delete mode 100644 labellerr/__pycache__/client.cpython-310.pyc delete mode 100644 labellerr/__pycache__/exceptions.cpython-310.pyc create mode 100644 labellerr/async_client.py create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100644 tests/__pycache__/__init__.cpython-310.pyc delete mode 100644 tests/__pycache__/test_client.cpython-310.pyc delete mode 100644 tests/__pycache__/test_client.cpython-312.pyc diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..1405387 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 160 +extend-ignore = E203, W503, E402 +exclude = .git,__pycache__,.venv,build,dist,venv \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d6bf56b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + +env: + PYTHON_VERSION: '3.9' + +jobs: + test: + name: Test Suite + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run linting + run: | + make lint + - name: Run linting + run: | + make format + - name: Run tests + run: | + make test \ No newline at end of file diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 0000000..8fc83ff --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,73 @@ +name: Claude Code Review + +on: + issue_comment: + types: [created] + +jobs: + claude-review: + # Only run when @claude is mentioned in a PR comment + if: | + github.event.issue.pull_request && + contains(github.event.comment.body, '@claude') + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@beta + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + + # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4) + # model: "claude-opus-4-20250514" + + # Direct prompt for manual review (triggered by @claude mention) + direct_prompt: | + Please review this pull request and provide feedback on: + - Code quality and best practices + - Logical issues + - Potential bugs or issues + - Performance considerations and optimization + - Security concerns + - Test coverage + - Code repeatability + - Over engineering + + Be constructive and helpful in your feedback. + + # Optional: Customize review based on file types + # direct_prompt: | + # Review this PR focusing on: + # - For TypeScript files: Type safety and proper interface usage + # - For API endpoints: Security, input validation, and error handling + # - For React components: Performance, accessibility, and best practices + # - For tests: Coverage, edge cases, and test quality + + # Optional: Different prompts for different authors + # direct_prompt: | + # ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && + # 'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' || + # 'Please provide a thorough code review focusing on our coding standards and best practices.' }} + + # Optional: Add specific tools for running tests or linting + # allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)" + + # Optional: Skip review for certain conditions + # if: | + # !contains(github.event.pull_request.title, '[skip-review]') && + # !contains(github.event.pull_request.title, '[WIP]') diff --git a/.gitignore b/.gitignore index ee5ab5e..a449859 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,7 @@ build/ dist/ *.egg-info/ *.pyc - if file == '.DS_Store': - continue +.DS_Store +.claude +.idea +tests/test_data diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..877d9fb --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +.PHONY: help install clean test lint format version info + +SOURCE_DIR := labellerr +PYTHON := python3 +PIP := pip3 + +help: + @echo "Labellerr SDK - Simple Development Commands" + @echo "==========================================" + @echo "" + @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?##/ { printf " %-15s %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + +install: + $(PIP) install -e . + +install-dev: + $(PIP) install -e ".[dev]" + +clean: + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete + find . -type d -name "*.egg-info" -exec rm -rf {} + + rm -rf build/ dist/ .coverage .pytest_cache/ .mypy_cache/ + +test: + $(PYTHON) -m pytest tests/ -v + +lint: + flake8 . + +format: + black . + +build: ## Build package + $(PYTHON) -m build + +version: + @grep '^version = ' pyproject.toml | cut -d'"' -f2 | sed 's/^/Current version: /' || echo "Version not found" + +info: + @echo "Python: $(shell $(PYTHON) --version)" + @echo "Working directory: $(shell pwd)" + @echo "Git branch: $(shell git branch --show-current 2>/dev/null || echo 'Not a git repository')" + @make version \ No newline at end of file diff --git a/labellerr/__init__.py b/labellerr/__init__.py index d1df3b7..fdc2454 100644 --- a/labellerr/__init__.py +++ b/labellerr/__init__.py @@ -1,4 +1,25 @@ -# labellerr/__init__.py +"""Labellerr SDK - Python client for Labellerr API.""" -__version__ = "0.1.0" +from .async_client import AsyncLabellerrClient +from .client import LabellerrClient +from .exceptions import LabellerrError +# Get version from package metadata +try: + from importlib.metadata import version + + __version__ = version("labellerr-sdk") +except ImportError: + # Python < 3.8 + from importlib_metadata import version + + __version__ = version("labellerr-sdk") +except Exception: + __version__ = "unknown" + +__all__ = [ + "__version__", + "LabellerrClient", + "AsyncLabellerrClient", + "LabellerrError", +] diff --git a/labellerr/__pycache__/__init__.cpython-310.pyc b/labellerr/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index fde0b577c5eee172ea00514c923e5cb4f383a3f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 164 zcmd1j<>g`kg0nuU83sW5F^Gc<7=auIATH(r5-AK(3@MDk44O<;tOk09dIo-)jJLSs zqFjn>1nKzSB0%t4iuNPMRil8>i`R?12*q^jyWT}@k>m08dhGu2F5<>^&j%4Dlq`8KKszV(INqFFT;^VNJ% zu23!D-t0ngu~aP~PqPdh_f`AkIEUkMwH%bqRr^tgxiGLeSRF)OzB*(Ts>4>XI%1Wo zqjM>%@3ZOZSn$0C-({R{wfd{$I3DLdqJyrd4-Y=`*Q`ZLi_UQ`4`dkxr!j@{{%RjfI7VZJ)61wmqTw zLuW78jkFn;3P9Q!>{bJ+g#v5 zer9PIrI%Z7GnM+2RP%*QDmBMOQ>ouO*8F@P?drZh@3dR2XQo|~!_16x+0QJs<|eXL z!){z|Hm+0)cCGF>&AC>?s`kw+G#f3q7F;UMB(9nDnVH5CyO#B{&Bgj$!_QuBT8*}! ztuI^6wr^VPndL>??(1&jId?+$^vU_9IWLQ}F^3bhixamwJByRWrFTpEE~HoLmwDM@ zzKc6Ees0Nb&o^dV-%vgBjiu%D3(c8{BHC^(ExXmMU1y`Y^BBTLz2zHDV*xi*P0MaC zS?w#WYQ}NhYNok3Q!PGaZhSS}K>YHjD%~fXXhJ|^Bvd&~7 zy8ZA{-L5ZI>PyYq#m36v%2{-irEwz9QC~*Kao0@UZMIvg&ulhvpVL6qZhWgczvJEB z_yyNpat=>Tq2bAekgz8~xzn7uL3jKD4^(}=dA{y6YRmQl3h0=pv-Wn3ki7%Rt(3hR zU+WJ}U1%>hrkv(VeadY(9Q>&>kA3L$ihH5mnu?m4QtxSUX~j2dwI&9zR-3~84tcJT zGtz~$w>5fKY8)rSg6sWDcH>aJ)oQykB+iVEyTU&c2RVG?4So^_ZfZXL8F258hE(fh zkY+l_c_p{1^S#dVld*J?!j0@|*45{;9levaaxZDCM#q>pEc0T*&qold!ZB%2P1=0@gP`uTLnSS#NIe;!Yv6&*9C?0qSE<&QU9nJHG& z2CY|?mloP}t8(t?C*_rKp3XHdH(Hh1=0c;=sxLM+nJ>InlWXd|FEkyu(w^tdA1h3ji`TzF`Jg{mMUg9oF#DZMrGD+FG}%P?#<4E0NAnRZvwJ??%7XVRhTgQ| zDF;^BUbbf%zR|XubIsPRw4b|x#i(IBuYCdU-MdFtsmff#6&}z%*y!5f%KE4oj;Vo){12f`e3?<)x*z?Z#}xZd_W%?5?^=YIy01M?xW~*ug^CaQ&jH5DTQ$z$m%4 zuP-m7t}N4c`?-bo93Fhm&o`dqng;e@520$CT+Al8nK12LY%3p)w!NR{x!^_k8H|H{ zfW;p|QZ0rLgdPh)tE9qvM$x018pN7%67D78swX?jODbw ztCQ&%R_;hDByOCUFJ)G9ogBzJ5AuH4%~=JUnOi{MAm1EG9b{Ydejy@@<7%Eem4)^U zrVPgv9KRjYL{#z}6P#KDqPOJS1x>#b8r_$q(STg1M&emE!QzS#(0YN zkEd>G>C~nD9YUszcPN^ir|tF(7!>Aiy>h^2fllmKW>ivC;$- zztC)5bS6w+Ydd%~b@ze-1zd;h$M8tL&UMJw0bU%N%W!o|aHrd@0s8}z2Q^OQg*7RD zBvZ|)@m~8;R+E)i5%4XxW4{mC>K&fo5z$7>oT~k`8rBUyR!tu15-vNOD|*%hBPyiF zkd~0@nzy_6#r53V1qk^JoNmH>eiH}YlP=((o4%&mrH-}+JdIbTwepgixu!Xzu5M}M zAuf2(wX~JFsM))brwd=<>)x@NagF)hnu*t$b{@pFEMdtj>MmZ%%ax>-a{~;bZ!{Yti@|R|v10 zjQ#wH2-$p#G#~(NA{X-x`$Me#DJFz4ihZOTJq*GxsqQNl|2zshTpF}uK_hnnkI>At znJIy_^P|00!m*JqYa;+L<|ni_y&XM0olI8P#WKj_$?Jt>v~Ums7A1jz9IcG2xtTSc zJRQWDTg?E%WbrWqIYeL8U4Rk57LYmQ5RmwMUgVH>xdlibI72N3d>3)QVlCOVQUF=r zjHvkO24-~QvhezVx?&t3J&_WIqniS%yVABVI!pDL#wL^+U5h7gf>z%dbq6|?d+Q8Y7(vJpnlG!@Kg0QzA{+6F5Hiaxjfy|19 z{xI5bMv&~w>j2*Zz27Jsh4fYel`RkSskxO0Y#BK+%G86A%ssWIIqbV8MW@ zQYls{(d>+Fi57@s6{`u&Q@aJc3`G=9psB#Z!n&MgC?Fjtl)c+a&{B14cD%h)LEX+& z*s?QKx2*Taq;hnWQo%uk{j8s|)qg zAug(kIeb5-!EQAMm8ZkUKQ51dy1qh%#o>ytpFVZw>@6h%_&VpXcQ7i9nrEAH%j)T{ zd~i6i#Hz=iU0zsNnZ&J>8D9G(CNE=7zVfE@kqF!9wQems?E;{$11$Azh8LVgnRQdKa}(^)$6a z^O&ALg^~_gl9nr_D``EWK`aM%(2Vs`zMLM?Os#-*Z3OEcIffF?3q~2D4Fq+rbsNO= zP21@bZu-z(gp+>*FVXu_C%p!)aWn0vLDBs#6sIm1JB4yHZXnWtI{@)2eq!n4DI7uk zK{=}8z+;#P6iwb@;*+;>)KuRvASP*60kZvvy(0OHDv7)zmdo5Tsn(-{htGlTP2k}? z+ApC#Q=#Di`UsGn>CM0&Ce%l?!R5l zZr@rN^$b8H<7f<8LpL>5FI0uYswHdWhJh9eZvXs%H45R+=oG9m@Wo=MI6r7@!RUQK zj<$jy{IwJqR>JRV26FW`B%TRXsEcLoYcDQoS*6Ya)xl&ngp zWZiQ^3pk{<@`us$46O}Ydpc%Gmt4q0zipd4^l1dwCx->BRU zS2sl-hoPuX@JD32b?ATt{;0}?Y=fG7a?PNIc#+?n*Y?yE>f8YAshLAh0cBBJLiH)n z0KoOGls+43+R-Ov5dAlXDh_^k)98kt%ToSlL@ecB6yT)WM z6DnG68R}A<88L4m5V6UDr=eci&l7@NZ|`luIKX+({lk@88M(nP&bAl8JZmC5yR3q8 zwRd1+6;dvi7owW{QgA3*W`8(5vl`Bf-4t!34-P1rlL^CKVH+Nks5Qa-s{iMCVTMVA z$sCglOqxu(S-j$RAH@Yf7pxW-1_AfX^tQ#%phh}=`iT+2U4G%QV`q=m&OUqk#2KYF z>cMJ+=)LNuk1+2XTiq2xdo_egAcOuQ7y;3&+KVhs6_vfjM37$0*ZUO6D~E%oCbi!m z)uo#72ZGmx+EtEC%|!wN@~zqA{XPtx&hmn5+suhZqjyOtt04*d`g1OLq05(C!Y-WYM|sj|)fC_zp} z%4|oD*j{j#E;;*W>wgswgwDb%0}$^2f?o4Q-Fo7yw-1xcOVZcB@Y1Y zw$=RAyeM_HC)C%Q$%5wrPhAK6a3jx-(52F&Bv__M>M^HG2!5OEXdmId@l?bl7(DD* z?1n52N4=r_fYOUicq2eXz5wZa;iy*%AuBMRg6oN;;NzI{_GKh~=J69}mFW5QeA3$y zGeV4`$oVlVoSqo4PqAKMeHm6>ICK2e=@YdlpEz?CJ{9d|s|W5TR-hinM#8jX?pBD^ zFL0L##^0uG1{xp^YcCfQcC{IT?WasyUswqF)d3K_^R1|;kT*pyL){ENCJ?Izoy`5f zu}S_lklzH$W)oO;gSNM)#V*xN2`%%(;c~u!#7eJb@X_a>Wpz_)=??HLG`MFznpp+8 zlfZHIp-)l8qd`?~T!+$qHSg_n^%qmK;w1EfNj>m5RP@)tz?=u3NnIL(9BXv)Vm9V| zIUcWq!C3k?r=SYbI|Y2}ED5A(u6zunUIeKRxp`n?Dt-*N&?yR%1$#@Uo=H6mmbF^E zTD&x*zDfL-ij=TB+1U&X0H*il2)D(IfLO?daJLW$w!}vc92r6lr~~1(0=1wlgd15z z0%Is9L9`KC#KM~@!C!(++(wrh@d;TaZ^I>S*IhnJ&wZi|?b38;c%?c%oZf%74as+< zI;oGDtLf{hH7qqZ(;rU3WTIKnrr%0W!=h|9T9!lakz1OtO+M&n;1Psuoh`-Y2S0yg zuF+~dw`4!Ehza~>Y(o3Q)|B^D@8ZYFCr?ld#*P&!2dEkbKe)zMFnL6fmGj7CRJYCU zua}iNihfsw%=TYH=JZ5Sw5+~+wC?CKrj?L_LIDus@cGh zBc){Pn+r zF4jOL8L^451|J6@Yv_<6)IXKF4%OIl%I>>lc)Q({kUt0-QME*NL)P}3gL-z|- zk%nG3V`T%P=QoMo0MTbxJ_w?xA!o=nEU*<2ea_7V#+;bw;h}QXxMW!JeI+gIVlHu~ zdAD~`2nMrA3$w_3fBf|`3oebDunNo@W#HRrmZ|jGS(G(AT2a*-D@@jy#I*WZ zo>q|fdlR&Jy9ZfG5~&H(3YAPG(IkO}WLn60@A^6hYlhq~{4;S7sh|D;C*IkF3R^t9 zKtWi(k!nQsP%mX@f2V{&BQ3BzRoL0U{$gOchgu49K!(gd!{KmSaFAh^;3ZSHwgYi1b-)$ zjxt-ueMv6ucQlg*u$Im?-Ih}Cp-@F$v+Vk;>x8FpL2E#p1g);vfD;&9bmW_XZu`42 zU+z{p;qBZ6bO&;VBUaJP6v?+o)?7}MLV>D>1jbgLTS|Kp>jXsjKfsc=h15GqpM=z* z^aD6viv+0eBV|$~(DKS1!svq2Lz}_rTp7!3UB{*6a($d5R6I(^E>t{LAHsk{gnTYxdx{-h zD|0YjHsFE-q$}SG+fzcBmjV<|xNQG@B-OsC>*BH{s@jOV zRQqItL1!%Xy@X$A)}}cz0S6+~<8Ke%N&K|6Xqx^4nav|;x0K(Z>0mUlFTH6PN%>vN z4?!c@t3xy)!4Jhu=Yd|PbB8v&U_lmbbHI+{<6K1&+N4&&V%7i!z)oQ2!4YjdIwf%U zf^KosR+a<`mSezsYcBwRf&9J5$FeN>I`XFo48T~Xke`q8b^Com`J>2(T^n9j#WPSH zLJBW=zaG)UhiOs3B&Hi)Xuu2Xl7aDKBVC8(0z2cK34C)yN?1k0?dWYNv%4Pn3N@8A zBqn%&Jn{C!29#9mkE%^LJvK^zN3bGodQ}8nZ}+CxB8}Zc%Gv3Icd95V{9xq@Hc^0o zg_{O?`a>6}2VrHvgiWc|&)l)z8Q8qqt%VhOXFW(?vWM=Xfrq?j?pSH4z`UEz&WVkQ zwlVbNA=-)F&8~B@!?aKLRhzCy_Kz|7aVDQ*((N|sZxyo^CSh_oXuNvVt1KVG_8VS98D$*v$d-px;c}aHqsE z98o~(W(yVq3&vm&a|RCJn*EGiHF?!4jHl@SEk@rW2>%(BElFA8l0LBnl*K@rh1s@X z^^5aLYOUBQbm*K}r1NE`h`Qi`3n`-nEHcn3Y)L`F&$;FKervFU9s;`z-~*#DOu;y& zAs9y4QJ|K-PT$%PdQkR^@Cb(I5yFg7j)S#jwf|~A@dR>5 zt*scTFUk=O!k?F;ZNNOAhekqbmz?c^q{2J4q1+cyE4FQQ%GRzxQYq75Y*k=19*})d zV;%ImwncVEZN<~y(-}}bNUL`DV8mYU?Hdg1-NTW#2Po7{HDy?f$OLoFt@3dKYp_%Lqdt07e0 zxHTq%SUqaYfRH!dpf1F`F;IKPoq*=gUaSYZwbXIw3)J0Nf}1)v~rTdUn&Bh`?HL$9n-1SNlu*$}>u4xB^dS@UE4+C!1vma#gVI~wsePh0T z9&M~ooZf`u5?)`El@GShL+F5;@dLPTVyAB?MYnPo9y~TZed_G7vrn9wuAMyf^i#*q z+VsRqD72pD>9?{;Q@o8atn@;_D^}Vq;!W9l;Ex0oAlQ1K_AVuC_VHkGNm3f{As&e_ zUqJJ#JY7Ii-P$!uH9oC!Dv4!wr8yr|t1oX-()&N+(LY84Ke1q<;~iq6JMa_Zw=(y? zk0r`t4~S-pZ-y`2AKb9@D)t2M`$9SvJDS4jud>;m^?qBpkE8ox9I((l7GsD08(NVH zy8^wz;0aJp@6bv}k(Z&a-As?*D)v%Pnb$|fUbyKC-|Je=_$oaLsDlSifX~22Y+u6m zPX3qSOMuNZ@DzxY3kRa9e_Yv^Z{OW6R6*T5HGo7v&XY)9Oud+XQF}4-qCT6ofFfQn zX};qQo13Xm_Yq^jScknf&tS8!mC<1nx|T`9=<`s@*?k2DHU2+~k44j&UVk;Y-W=m- z1q%)UpqgV&&q71qO@jUm%6TWg=1c0T4B>rv->o zB1{0$>$-SKXRp&$#JEndn7giksHT7@jXw4x3W$mUC=&zG>yRP=L_eZ{D6d99l)HTh zMB%EUfT*<4CoPm^hY-Cb10Z@`2N2C(ClECRh>BHOfG9vGfM`~Ls1^WG?Yf4358xv} zl*a-@a{@$708mq)Xg&g)2N{?&p2P!n*Ho{PT;>UpnK~` zhS zeVj+cC-%=WAr0C;$K=kcwtvkcL>2a{On#mT!9wiM4(oNdKRZCDEZ9|+jTZLx1GfQ# z!P?!03C;Zq!W=y-76C6?dE}8tDgt&Gd?biy;_airLC_ZZl+F^beiL?(gN0d6Ws?7| zQS-EZe+;#tl58)C1Yo0a9&v2_dy0un5rUu?Re znRgRNYX2t7nT;TLj~eU$f;_vz3-{k`9~*oPgkU4ZQqxTDd&_(8*Za6#R(KLbgzO%} ziTG70M2F=$LUgd{a;NmBh;I8vMCbivq=tPwh{X|5GgJn|Y)Pd72~}Q11o~pBJgNph zRRB1SibGQg{^A!i8N8QC1%IfxRXMjoK(kG5ppRKZ)XogXAo??K6v3EZ#fb|qlzG~- z;m|a%;e${=Smc;XAJz=MxpfP^C-c;+W%12LCbyb~mP*4yx1YNX>HXS)pRHKEGo6g+ z!ng+=mNi-43Kq6?0g!0+kq$go&S2v)VF|Z(VbgJ6u(a*QvQ%DaU{T77Wt%p!GAv+F ztw#&nU@|9Il(JW|gbKn}F~}wFLRk#EkM`+(s~`v%oDl$MHZ!ZurJ(?(85(L|QSx$s5@Kn= z{UZg#G91M_`3~H~U&^kQp241=Q2cC}5I@^VX4L^X-At|axn*mp(cpQoFk+1%AN@076qQ5-QQu} z^Gte}^m~v8Q@k+Y&4AHnP?vn%PtPj9R#|m^g*Cs*Y(uT*o7%|tygiJ1iyZUAs7NdI z;Mj~(zv&260Hs^7IY%u8RJrLn^Q$}FFiUWL)w(b$)N2X9orin*@7s5&Ykwz)njhi8 zKLlbAI51pBtrXl|OD?SGA>>WLdGB9>U+$$@3l5fZM8C8Z+R}hs&+1gbe+7LCLtD}vN;abrR z_MbnWzI6K1F7Mj_YqE@Rh>)0Y`w^oP!{^Ws0D<1-W$!wA)h|Xx7{1Ut9gY+)7fs0x z9cI0eV4F{$HHwx#2aQ7?cy8AIK{?t2ZO0D+@|fISBsp87Z9aY07L@xO`ZjKD1Lxiz zZSz6dz%kIA$DYuYFM=;~tIx1IxHe?%aECkaHsEMTuT#mK078Y^e|An!=+z<|MiISC zw((Vmf|KMfzP=z&*)X71m(7w5#~Y9BVij1tX&8aMyM?XW-qy$f)_Y|~P#eH649PuL z6B>s9h1ulo4-GqAUD#mMvHyUDqivXB2jg-ug4b=kzT)i;D^z9T62V3uyMvnG7Qe6n zKcU26V0*sU46CKkTL{Js$J;rD*mP5>b5qpKcbSotTIA!XQ9O&FgNS((6%k(Pc(A+v zEO*xvyT6U*W0H66j@1Ur(Ip5trn&)b!HBYz3_p45_^~J9&VTyU(`Q3#$tgDS1QNgd z@);Oa&YkfM<)qZRhw&Ki65ty@fiTfayznDTew4|#Fd>^#wvJ#UP{cp0;Z#yDKcO!W zcpl?mQ40vBjHUbGz^g~a;!+QLGy}GH0qub)LcC9Y=!Za&o&Ntkl6?SgX`j$^VU8NC z7jG7TnMV-wDPWgxXom5XLb2D|sE|$g8f^^gg#{1}63Jf8P$Uii25^waM>r$RJW;q8 z=oJy4&U8gI?dWshjk3Et2cgUWpCOM?PRH;&va%4=3UoRql7#lt0B6AtaGETzTRYb& zUB<}6JpfUk!tYha(I~CKDH#HpxI+LH_JQ{e0NWvwMyKB@$&UIy9OZfR3U?TlFCBxk za#^@_2?ma`cPx@GkIU22CnbWHOep??#?#O{#7OcvB3^|4aFLPf{Kso^#oB8Ak z%}{Bk1ik@kGDpN?a} z51N8wA0UhOM*$9i0tWN$D7@)1*BS3&6zf_LY&BTd>077S``q+(xZGf6*RX2uLn-BH zhd7P6`WS2=2waDnwyYxXQHQJddr(F>>PW;h*~ohwd3l%gC79#V!Ao%VMY}2c+pMDc zMhS7*k$hkP$yZpQPgcpY^s@;EUx^%Z1O2wjM^e|dHJwKYp9h5ojt0@!Ec&sE8i!oE zBuksvmNP6W55pSa_jPSPhth~vfpk<-+==5c$ZK0}piaHp5()}nsU_B6^lf+#JV3>bi3-5y5c+mUqi1VDX!4M#Bh)ZNJ`4}=;x$?=nY$n7w-^IuH1pD&d;|jLnxf$fVa=wiXnu24n)iAKGi&uBB9Wl!E z-giO;o$D=dlR7qE=u$i=x~fPKwrjsEfv~=D{@7oF~2AvVc`YNtYZ11#q!g z8TQ9eTsC`i-{p*}bd=|q*+v4O8^xheH;IsXD`U&zB%rKE7ugpq%Lvz5w^n|eD+`@L zHF4N5wXw7T1tipugkB67^hmFC$&lXMYgALKb!gLwO&MJcbgBID)DnXI48FyrfKg8` zL&-d$O`jD#W}tJQk62 zf?tJq2}pp=<}Q%LDr?(uItX$VAftq{)xru0a0-Q#w-#}_H{Mm82x(UKOTqEW+qYqN zA@+ZRs>N3$#dH{|fFKU>?E&KK?SMNFN)YlO5i2FwxhxZ21T?XM6B;zCH)0R#jo)IoG!I;Rt@lH0pTNWf4p-Vqw1tW!8@_=$viCV_r zc`fijFT(ZZy@~!8UQRpj$M|mJdqLl}yCc`Ax`JOk0%H)|jxtA`TRX31R{Pz8JGQok zad{A6V=F}1T`#3SUHg<4diTxh1ja8wT$Wiopv)5(ebU<`gD7=vx@jj=m53V}f>GV$aeLJQ8TT}B=qIltD)V!VD0s5#OfNs;?PVZsttt-mB(Ee@k4nq2E4Yij9|ZW;-Zb; z09xR(z&Mu3+vwZKTfTukO1wP==aHf0PFsw_g&^|E33e)mTaS}rCj*~Ntj_{&lP%q8 zH!{2y>-Zq94}SYW{T2l8M^I+?SU3kOanK)}046I9dpJ(fsM}Y-gS!w9!3zB~1d9@9 z%>_|OI+zHMJmGuzH?>Rn*_jyg0NlQeg7%*x0URnO23!@Y;DTQ`!pEoU(j)hx0YOEG zO7dv$3KG}?5+O+(iU+}qk3m~pop=9JjBkYb5J@SbK9PViiOxXp@&fz$91_GUQnBVF zO3a2()Z?eSLQyMs=eCVs#PN0`pfzDsVu%G$sScwJg8pQ9ygIxmRuINU6%gN2c*i(| zI74S8-i^FDiROy>hv1t((`dkXOWq}SQFw-$9Nu7}JmZ%ft~B|wyv*c7v);&4k8Zrx z*PeRR%T%6vRBccjRrsAhJEcZUnpV*qc5J{~F;!K(_u#Qb7z$~Vn-WNq!OQ|SW7Hc5 z;UJnAeN1N=UoU_B^f^2;LVQVeAY0SShm@0Ce8krwF%GxE1^hkcf9sM+afB^E!_;*atQlnH?| zL65XiI6|pKrYU%@0{uox)5hL1-ueb6YgDOH{gobm@WI2CGb;S7H>$=Z7|9R;dqL1 zm181K>{1~5MUKo#CXX|Dnzv?~4rV_>^vIO)^`~p6K4jnIWr4@9YXdPd#zAcKXEAHOBinbK$DB zHc(gKBv+G+>QL8dO=1&0#eT0b$w~K=O_#h-?y>(TFCm<4U{V(86P7t8Q`p~$tJ^q% ze+~ya_CX9ufTDrS5ai||`4>}!hCqry`QRWr$&~Y8tMl=ogQNGaeOFJcoU(}2@OVIQ}dVrLUAorRfQp6C0 zgMUK%n)a1l>!o2_N3chffuw;}Q9?Wy-|{y2>)d&b02A25vH2b(kkh9pw(*C(_!eq4 zzfgm>rnQVfS2e#>!|SUr1Xpsk8h&lBR(9?xz+rrc?J286pok^rRU`CiQURc;hWjmk1(OMr7UeC z;gZb#A!rbSnHM?*inFGonJ@8f)X9+Xb@q-1k_n;=WDpR-ccf|;tznH9!X%_GCz zcbzR&!{kmB+v_m%ZKwS;p})p2ib+Hx+V|FJ;EtC&$$jhZuOS=Z3m0Du8ZZ4K6Cg~3 EAFr2MO#lD@ diff --git a/labellerr/async_client.py b/labellerr/async_client.py new file mode 100644 index 0000000..e4cd26d --- /dev/null +++ b/labellerr/async_client.py @@ -0,0 +1,328 @@ +# labellerr/async_client.py + +import asyncio +import logging +import os +import uuid +from typing import Any, Dict, List, Optional + +import aiofiles +import aiohttp + +from . import constants +from .exceptions import LabellerrError + + +class AsyncLabellerrClient: + """ + Async client for interacting with the Labellerr API using aiohttp for better performance. + """ + + def __init__(self, api_key: str, api_secret: str, connector_limit: int = 100): + """ + Initializes the AsyncLabellerrClient with API credentials. + + :param api_key: The API key for authentication. + :param api_secret: The API secret for authentication. + :param connector_limit: Maximum number of connections in the pool + """ + self.api_key = api_key + self.api_secret = api_secret + self.base_url = constants.BASE_URL + self._session: Optional[aiohttp.ClientSession] = None + self._connector_limit = connector_limit + + async def __aenter__(self): + """Async context manager entry.""" + await self._ensure_session() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + + async def _ensure_session(self): + """Ensure aiohttp session is created with connection pooling.""" + if self._session is None or self._session.closed: + connector = aiohttp.TCPConnector( + limit=self._connector_limit, + limit_per_host=20, + keepalive_timeout=30, + enable_cleanup_closed=True, + ) + timeout = aiohttp.ClientTimeout(total=300, connect=30) + self._session = aiohttp.ClientSession( + connector=connector, + timeout=timeout, + headers={"User-Agent": "Labellerr-SDK-Async/1.0"}, + ) + + async def close(self): + """Close the aiohttp session and cleanup resources.""" + if self._session and not self._session.closed: + await self._session.close() + + def _build_headers( + self, + client_id: Optional[str] = None, + extra_headers: Optional[Dict[str, str]] = None, + ) -> Dict[str, str]: + """ + Builds standard headers for API requests. + + :param client_id: Optional client ID to include in headers + :param extra_headers: Optional dictionary of additional headers + :return: Dictionary of headers + """ + headers = { + "api_key": self.api_key, + "api_secret": self.api_secret, + "source": "sdk-async", + "origin": constants.ALLOWED_ORIGINS, + } + + if client_id: + headers["client_id"] = str(client_id) + + if extra_headers: + headers.update(extra_headers) + + return headers + + async def _handle_response( + self, response: aiohttp.ClientResponse, request_id: Optional[str] = None + ) -> Dict[str, Any]: + """ + Async standardized response handling. + + :param response: aiohttp ClientResponse object + :param request_id: Optional request tracking ID + :return: JSON response data for successful requests + :raises LabellerrError: For non-successful responses + """ + if response.status in [200, 201]: + try: + return await response.json() + except Exception: + text = await response.text() + raise LabellerrError(f"Expected JSON response but got: {text}") + elif 400 <= response.status < 500: + try: + error_data = await response.json() + raise LabellerrError({"error": error_data, "code": response.status}) + except Exception: + text = await response.text() + raise LabellerrError({"error": text, "code": response.status}) + else: + text = await response.text() + raise LabellerrError( + { + "status": "internal server error", + "message": "Please contact support with the request tracking id", + "request_id": request_id or str(uuid.uuid4()), + "error": text, + } + ) + + async def get_direct_upload_url( + self, file_name: str, client_id: str, purpose: str = "pre-annotations" + ) -> str: + """ + Async version of get_direct_upload_url. + """ + await self._ensure_session() + + url = f"{constants.BASE_URL}/connectors/direct-upload-url" + params = {"client_id": client_id, "purpose": purpose, "file_name": file_name} + headers = self._build_headers(client_id=client_id) + + try: + async with self._session.get( + url, params=params, headers=headers + ) as response: + response_data = await self._handle_response(response) + return response_data["response"] + except Exception as e: + logging.exception(f"Error getting direct upload url: {e}") + raise + + async def connect_local_files( + self, client_id: str, file_names: List[str], connection_id: Optional[str] = None + ) -> Dict[str, Any]: + """ + Async version of connect_local_files. + """ + await self._ensure_session() + + url = f"{constants.BASE_URL}/connectors/connect/local" + params = {"client_id": client_id} + headers = self._build_headers(client_id=client_id) + + body = {"file_names": file_names} + if connection_id is not None: + body["temporary_connection_id"] = connection_id + + async with self._session.post( + url, params=params, headers=headers, json=body + ) as response: + return await self._handle_response(response) + + async def upload_file_stream( + self, signed_url: str, file_path: str, chunk_size: int = 8192 + ) -> bool: + """ + Async streaming file upload to minimize memory usage. + + :param signed_url: GCS signed URL for upload + :param file_path: Local file path to upload + :param chunk_size: Size of chunks to read + :return: True on success + """ + await self._ensure_session() + + file_size = os.path.getsize(file_path) + headers = { + "Content-Type": "application/octet-stream", + "Content-Length": str(file_size), + } + + async with aiofiles.open(file_path, "rb") as f: + async with self._session.put( + signed_url, headers=headers, data=f + ) as response: + if response.status not in [200, 201]: + text = await response.text() + raise LabellerrError(f"Upload failed: {response.status} - {text}") + return True + + async def upload_files_batch( + self, client_id: str, files_list: List[str], batch_size: int = 5 + ) -> str: + """ + Async batch file upload with concurrency control. + + :param client_id: The ID of the client + :param files_list: List of file paths to upload + :param batch_size: Number of concurrent uploads + :return: Connection ID + """ + if isinstance(files_list, str): + files_list = files_list.split(",") + elif not isinstance(files_list, list): + raise LabellerrError( + "files_list must be either a list or a comma-separated string" + ) + + if len(files_list) == 0: + raise LabellerrError("No files to upload") + + # Validate files exist + for file_path in files_list: + if not os.path.exists(file_path): + raise LabellerrError(f"File does not exist: {file_path}") + if not os.path.isfile(file_path): + raise LabellerrError(f"Path is not a file: {file_path}") + + # Get upload URLs and connection ID + file_names = [os.path.basename(f) for f in files_list] + response = await self.connect_local_files(client_id, file_names) + + connection_id = response["response"]["temporary_connection_id"] + resumable_upload_links = response["response"]["resumable_upload_links"] + + # Create semaphore for concurrency control + semaphore = asyncio.Semaphore(batch_size) + + async def upload_single_file(file_path: str): + async with semaphore: + file_name = os.path.basename(file_path) + signed_url = resumable_upload_links[file_name] + return await self.upload_file_stream(signed_url, file_path) + + # Upload files concurrently + tasks = [upload_single_file(file_path) for file_path in files_list] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Check for errors + failed_files = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + failed_files.append((files_list[i], str(result))) + + if failed_files: + error_msg = ( + f"Failed to upload {len(failed_files)} files: {failed_files[:3]}" + ) + if len(failed_files) > 3: + error_msg += f"... and {len(failed_files) - 3} more" + raise LabellerrError(error_msg) + + return connection_id + + async def get_dataset(self, workspace_id: str, dataset_id: str) -> Dict[str, Any]: + """ + Async version of get_dataset. + """ + await self._ensure_session() + + url = f"{constants.BASE_URL}/datasets/{dataset_id}" + params = {"client_id": workspace_id, "uuid": str(uuid.uuid4())} + headers = self._build_headers( + extra_headers={"Origin": constants.ALLOWED_ORIGINS} + ) + + async with self._session.get(url, params=params, headers=headers) as response: + return await self._handle_response(response) + + async def create_dataset( + self, + dataset_config: Dict[str, Any], + files_to_upload: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + Async version of create_dataset. + """ + await self._ensure_session() + + try: + # Validate data_type + if dataset_config.get("data_type") not in constants.DATA_TYPES: + raise LabellerrError( + f"Invalid data_type. Must be one of {constants.DATA_TYPES}" + ) + + connection_id = None + if files_to_upload is not None: + connection_id = await self.upload_files_batch( + client_id=dataset_config["client_id"], files_list=files_to_upload + ) + + unique_id = str(uuid.uuid4()) + url = f"{constants.BASE_URL}/datasets/create" + params = {"client_id": dataset_config["client_id"], "uuid": unique_id} + headers = self._build_headers( + client_id=dataset_config["client_id"], + extra_headers={"content-type": "application/json"}, + ) + + payload = { + "dataset_name": dataset_config["dataset_name"], + "dataset_description": dataset_config.get("dataset_description", ""), + "data_type": dataset_config["data_type"], + "connection_id": connection_id, + "path": "local", + "client_id": dataset_config["client_id"], + } + + async with self._session.post( + url, params=params, headers=headers, json=payload + ) as response: + response_data = await self._handle_response(response, unique_id) + dataset_id = response_data["response"]["dataset_id"] + return {"response": "success", "dataset_id": dataset_id} + + except LabellerrError as e: + logging.error(f"Failed to create dataset: {e}") + raise + + # Add more async methods as needed... diff --git a/labellerr/base/singleton.py b/labellerr/base/singleton.py index 53bbbc5..a4c429e 100644 --- a/labellerr/base/singleton.py +++ b/labellerr/base/singleton.py @@ -1,20 +1,19 @@ -import logging import threading -from functools import wraps + class Singleton: - __instance = None - __lock = None - - def __new__(cls, *args, **kwargs): - if cls.__lock is None: - cls.__lock = threading.Lock() - if cls.__instance is None: - with cls.__lock: + __instance = None + __lock = None + + def __new__(cls, *args, **kwargs): + if cls.__lock is None: + cls.__lock = threading.Lock() if cls.__instance is None: - cls.__instance = super().__new__(cls) - return cls.__instance - - def __init__(self, *args): - if type(self) is Singleton: - raise TypeError("Can't instantiate Singleton class") \ No newline at end of file + with cls.__lock: + if cls.__instance is None: + cls.__instance = super().__new__(cls) + return cls.__instance + + def __init__(self, *args): + if type(self) is Singleton: + raise TypeError("Can't instantiate Singleton class") diff --git a/labellerr/client.py b/labellerr/client.py index b92024a..f044015 100644 --- a/labellerr/client.py +++ b/labellerr/client.py @@ -1,61 +1,236 @@ # labellerr/client.py -import requests -import uuid -from .exceptions import LabellerrError +import concurrent.futures import json -import logging +import logging import os -from concurrent.futures import ThreadPoolExecutor, as_completed import time -from . import constants -from . import gcs -from . import utils -import concurrent.futures +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed from multiprocessing import cpu_count -FILE_BATCH_SIZE=15 * 1024 * 1024 -FILE_BATCH_COUNT=900 -TOTAL_FILES_SIZE_LIMIT_PER_DATASET=2.5*1024*1024*1024 -TOTAL_FILES_COUNT_LIMIT_PER_DATASET=2500 -ANNOTATION_FORMAT=['json', 'coco_json', 'csv', 'png'] -LOCAL_EXPORT_FORMAT=['json', 'coco_json', 'csv', 'png'] -LOCAL_EXPORT_STATUS=['review', 'r_assigned','client_review', 'cr_assigned','accepted'] - -# DATA TYPES: image, video, audio, document, text -DATA_TYPES=('image', 'video', 'audio', 'document', 'text') -DATA_TYPE_FILE_EXT = { - 'image': ['.jpg','.jpeg', '.png', '.tiff'], - 'video': ['.mp4'], - 'audio': ['.mp3', '.wav'], - 'document': ['.pdf'], - 'text': ['.txt'] -} - -SCOPE_LIST=['project','client','public'] -OPTION_TYPE_LIST=['input', 'radio', 'boolean', 'select', 'dropdown', 'stt', 'imc', 'BoundingBox', 'polygon', 'dot', 'audio'] +import requests +from . import constants, gcs, utils +from .exceptions import LabellerrError # python -m unittest discover -s tests --run # python setup.py sdist bdist_wheel -- build -create_dataset_parameters={} +create_dataset_parameters = {} + class LabellerrClient: """ A client for interacting with the Labellerr API. """ - def __init__(self, api_key, api_secret): + + def __init__( + self, + api_key, + api_secret, + enable_connection_pooling=True, + pool_connections=10, + pool_maxsize=20, + ): """ Initializes the LabellerrClient with API credentials. :param api_key: The API key for authentication. :param api_secret: The API secret for authentication. + :param enable_connection_pooling: Whether to enable connection pooling + :param pool_connections: Number of connection pools to cache + :param pool_maxsize: Maximum number of connections to save in the pool """ self.api_key = api_key self.api_secret = api_secret - self.base_url = "https://api.labellerr.com" + self.base_url = constants.BASE_URL + self._session = None + self._enable_pooling = enable_connection_pooling + self._pool_connections = pool_connections + self._pool_maxsize = pool_maxsize + + if enable_connection_pooling: + self._setup_session() + + def _setup_session(self): + """ + Set up requests session with connection pooling for better performance. + """ + try: + from requests.adapters import HTTPAdapter + from urllib3.util.retry import Retry + except ImportError: + # Fallback if urllib3 is not available + HTTPAdapter = None + Retry = None + + self._session = requests.Session() + + if HTTPAdapter and Retry: + # Configure retry strategy + retry_strategy = Retry( + total=3, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=[ + "HEAD", + "GET", + "PUT", + "DELETE", + "OPTIONS", + "TRACE", + "POST", + ], + backoff_factor=1, + ) + + # Configure connection pooling + adapter = HTTPAdapter( + pool_connections=self._pool_connections, + pool_maxsize=self._pool_maxsize, + max_retries=retry_strategy, + ) + + self._session.mount("http://", adapter) + self._session.mount("https://", adapter) + + def _make_request(self, method, url, **kwargs): + """ + Make HTTP request using session if available, otherwise use requests directly. + """ + # Set default timeout if not provided + kwargs.setdefault("timeout", (30, 300)) # connect, read + + if self._session: + return self._session.request(method, url, **kwargs) + else: + return requests.request(method, url, **kwargs) + + def close(self): + """ + Close the session and cleanup resources. + """ + if self._session: + self._session.close() + self._session = None + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.close() + + def _build_headers(self, client_id=None, extra_headers=None): + """ + Builds standard headers for API requests. + + :param client_id: Optional client ID to include in headers + :param extra_headers: Optional dictionary of additional headers + :return: Dictionary of headers + """ + headers = { + "api_key": self.api_key, + "api_secret": self.api_secret, + "source": "sdk", + "origin": constants.ALLOWED_ORIGINS, + } + + if client_id: + headers["client_id"] = str(client_id) + + if extra_headers: + headers.update(extra_headers) + + return headers + + def _handle_response(self, response, request_id=None, success_codes=None): + """ + Standardized response handling with consistent error patterns. + + :param response: requests.Response object + :param request_id: Optional request tracking ID + :param success_codes: Optional list of success status codes (default: [200, 201]) + :return: JSON response data for successful requests + :raises LabellerrError: For non-successful responses + """ + if success_codes is None: + success_codes = [200, 201] + + if response.status_code in success_codes: + try: + return response.json() + except ValueError: + # Handle cases where response is successful but not JSON + raise LabellerrError(f"Expected JSON response but got: {response.text}") + elif 400 <= response.status_code < 500: + try: + error_data = response.json() + raise LabellerrError( + {"error": error_data, "code": response.status_code} + ) + except ValueError: + raise LabellerrError( + {"error": response.text, "code": response.status_code} + ) + else: + raise LabellerrError( + { + "status": "internal server error", + "message": "Please contact support with the request tracking id", + "request_id": request_id or str(uuid.uuid4()), + } + ) + + def _handle_upload_response(self, response, request_id=None): + """ + Specialized error handling for upload operations that may have different success patterns. + + :param response: requests.Response object + :param request_id: Optional request tracking ID + :return: JSON response data for successful requests + :raises LabellerrError: For non-successful responses + """ + try: + response_data = response.json() + except ValueError: + raise LabellerrError(f"Failed to parse response: {response.text}") + + if response.status_code not in [200, 201]: + if response.status_code >= 400 and response.status_code < 500: + raise LabellerrError( + {"error": response_data, "code": response.status_code} + ) + elif response.status_code >= 500: + raise LabellerrError( + { + "status": "internal server error", + "message": "Please contact support with the request tracking id", + "request_id": request_id or str(uuid.uuid4()), + "error": response_data, + } + ) + return response_data - def get_direct_upload_url(self, file_name, client_id, purpose='pre-annotations'): + def _handle_gcs_response(self, response, operation_name="GCS operation"): + """ + Specialized error handling for Google Cloud Storage operations. + + :param response: requests.Response object + :param operation_name: Name of the operation for error messages + :return: True for successful operations + :raises LabellerrError: For non-successful responses + """ + expected_codes = [200, 201] if operation_name == "upload" else [200] + + if response.status_code in expected_codes: + return True + else: + raise LabellerrError( + f"{operation_name} failed: {response.status_code} - {response.text}" + ) + + def get_direct_upload_url(self, file_name, client_id, purpose="pre-annotations"): """ Get the direct upload URL for the given file names. @@ -64,27 +239,17 @@ def get_direct_upload_url(self, file_name, client_id, purpose='pre-annotations') :return: The response from the API. """ url = f"{constants.BASE_URL}/connectors/direct-upload-url?client_id={client_id}&purpose={purpose}&file_name={file_name}" - headers = { - 'client_id': client_id, - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://pro.labellerr.com' - } - - response = requests.get(url, headers=headers) - if response.status_code != 200: - tracking_id = str(uuid.uuid4()) - logging.exception(f"Error getting direct upload url: {response.text}") - raise LabellerrError({ - 'status': 'Internal server error', - 'status': 'Internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': tracking_id - }) - url = response.json()['response'] - return url - + headers = self._build_headers(client_id=client_id) + + response = self._make_request("GET", url, headers=headers) + + try: + response_data = self._handle_response(response, success_codes=[200]) + return response_data["response"] + except Exception as e: + logging.exception(f"Error getting direct upload url: {response.text} {e}") + raise + def connect_local_files(self, client_id, file_names, connection_id=None): """ Connects local files to the API. @@ -94,22 +259,14 @@ def connect_local_files(self, client_id, file_names, connection_id=None): :return: The response from the API. """ url = f"{constants.BASE_URL}/connectors/connect/local?client_id={client_id}" - headers = { - 'client_id': client_id, - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://pro.labellerr.com' - } - body = { - 'file_names': file_names - } + headers = self._build_headers(client_id=client_id) + + body = {"file_names": file_names} if connection_id is not None: - body['temporary_connection_id'] = connection_id - response = requests.post(url, headers=headers, json=body) - if response.status_code != 200: - raise LabellerrError("Internal server error. Please contact support. : " + response.text) - return response.json() + body["temporary_connection_id"] = connection_id + + response = self._make_request("POST", url, headers=headers, json=body) + return self._handle_response(response) def __process_batch(self, client_id, files_list, connection_id=None): """ @@ -120,14 +277,18 @@ def __process_batch(self, client_id, files_list, connection_id=None): for file_path in files_list: file_name = os.path.basename(file_path) files[file_name] = file_path - response = self.connect_local_files(client_id, list(files.keys()), connection_id) - resumable_upload_links = response['response']['resumable_upload_links'] + + response = self.connect_local_files( + client_id, list(files.keys()), connection_id + ) + resumable_upload_links = response["response"]["resumable_upload_links"] for file_name in resumable_upload_links.keys(): - gcs.upload_to_gcs_resumable(resumable_upload_links[file_name], files[file_name]) - + gcs.upload_to_gcs_resumable( + resumable_upload_links[file_name], files[file_name] + ) + return response - - + def upload_files(self, client_id, files_list): """ Uploads files to the API. @@ -142,9 +303,11 @@ def upload_files(self, client_id, files_list): try: # Convert string input to list if necessary if isinstance(files_list, str): - files_list = files_list.split(',') + files_list = files_list.split(",") elif not isinstance(files_list, list): - raise LabellerrError("files_list must be either a list or a comma-separated string") + raise LabellerrError( + "files_list must be either a list or a comma-separated string" + ) if len(files_list) == 0: raise LabellerrError("No files to upload") @@ -157,12 +320,13 @@ def upload_files(self, client_id, files_list): raise LabellerrError(f"Path is not a file: {file_path}") response = self.__process_batch(client_id, files_list) - connection_id = response['response']['temporary_connection_id'] + connection_id = response["response"]["temporary_connection_id"] return connection_id except Exception as e: logging.error(f"Failed to upload files : {str(e)}") raise LabellerrError(f"Failed to upload files : {str(e)}") + def get_dataset(self, workspace_id, dataset_id): """ Retrieves a dataset from the Labellerr API. @@ -173,19 +337,14 @@ def get_dataset(self, workspace_id, dataset_id): :return: The dataset as JSON. """ url = f"{constants.BASE_URL}/datasets/{dataset_id}?client_id={workspace_id}&uuid={str(uuid.uuid4())}" - headers = { - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'Origin': 'https://pro.labellerr.com' - } - response = requests.get(url, headers=headers) - if response.status_code != 200: - raise LabellerrError(f"Error {response.status_code}: {response.text}") - return response.json() + headers = self._build_headers( + extra_headers={"Origin": constants.ALLOWED_ORIGINS} + ) - def update_rotation_count(self): + response = self._make_request("GET", url, headers=headers) + return self._handle_response(response) + def update_rotation_count(self): """ Updates the rotation count for a project. @@ -195,14 +354,10 @@ def update_rotation_count(self): unique_id = str(uuid.uuid4()) url = f"{self.base_url}/projects/rotations/add?project_id={self.project_id}&client_id={self.client_id}&uuid={unique_id}" - headers = { - 'client_id': self.client_id, - 'content-type': 'application/json', - 'api_key': self.api_key, - 'api_secret': self.api_secret, - - 'origin': 'https://dev.labellerr.com' - } + headers = self._build_headers( + client_id=self.client_id, + extra_headers={"content-type": "application/json"}, + ) payload = json.dumps(self.rotation_config) print(f"Update Rotation Count Payload: {payload}") @@ -210,23 +365,16 @@ def update_rotation_count(self): response = requests.request("POST", url, headers=headers, data=payload) print("Rotation configuration updated successfully.") + self._handle_response(response, unique_id) - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) - - return {'msg': 'project rotation configuration updated'} + return {"msg": "project rotation configuration updated"} except LabellerrError as e: logging.error(f"Project rotation update config failed: {e}") raise - def create_dataset(self, dataset_config, files_to_upload=None, folder_to_upload=None): + def create_dataset( + self, dataset_config, files_to_upload=None, folder_to_upload=None + ): """ Creates an empty dataset. @@ -236,67 +384,63 @@ def create_dataset(self, dataset_config, files_to_upload=None, folder_to_upload= try: # Validate data_type - if dataset_config.get('data_type') not in DATA_TYPES: - raise LabellerrError(f"Invalid data_type. Must be one of {DATA_TYPES}") + if dataset_config.get("data_type") not in constants.DATA_TYPES: + raise LabellerrError( + f"Invalid data_type. Must be one of {constants.DATA_TYPES}" + ) unique_id = str(uuid.uuid4()) url = f"{constants.BASE_URL}/datasets/create?client_id={dataset_config['client_id']}&uuid={unique_id}" - headers = { - 'client_id': str(dataset_config['client_id']), - 'content-type': 'application/json', - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://pro.labellerr.com' - } + headers = self._build_headers( + client_id=dataset_config["client_id"], + extra_headers={"content-type": "application/json"}, + ) if files_to_upload is not None: try: connection_id = self.upload_files( - client_id=dataset_config['client_id'], - files_list=files_to_upload + client_id=dataset_config["client_id"], + files_list=files_to_upload, ) except Exception as e: raise LabellerrError(f"Failed to upload files to dataset: {str(e)}") elif folder_to_upload is not None: try: - result = self.upload_folder_files_to_dataset({ - 'client_id': dataset_config['client_id'], - 'folder_path': folder_to_upload, - 'data_type': dataset_config['data_type'] - }) - connection_id = result['connection_id'] + result = self.upload_folder_files_to_dataset( + { + "client_id": dataset_config["client_id"], + "folder_path": folder_to_upload, + "data_type": dataset_config["data_type"], + } + ) + connection_id = result["connection_id"] except Exception as e: - raise LabellerrError(f"Failed to upload folder files to dataset: {str(e)}") + raise LabellerrError( + f"Failed to upload folder files to dataset: {str(e)}" + ) payload = json.dumps( { - "dataset_name": dataset_config['dataset_name'], - "dataset_description": dataset_config.get('dataset_description', ''), - "data_type": dataset_config['data_type'], + "dataset_name": dataset_config["dataset_name"], + "dataset_description": dataset_config.get( + "dataset_description", "" + ), + "data_type": dataset_config["data_type"], "connection_id": connection_id, "path": "local", - "client_id": dataset_config['client_id'] + "client_id": dataset_config["client_id"], } ) response = requests.request("POST", url, headers=headers, data=payload) - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) - dataset_id = response.json()['response']['dataset_id'] - - return {'response': 'success','dataset_id':dataset_id} + response_data = self._handle_response(response, unique_id) + dataset_id = response_data["response"]["dataset_id"] + + return {"response": "success", "dataset_id": dataset_id} except LabellerrError as e: logging.error(f"Failed to create dataset: {e}") raise - def get_all_dataset(self,client_id,datatype,project_id,scope): + def get_all_dataset(self, client_id, datatype, project_id, scope): """ Retrieves a dataset by its ID. @@ -314,68 +458,68 @@ def get_all_dataset(self,client_id,datatype,project_id,scope): if not isinstance(scope, str): raise LabellerrError("scope must be a string") # scope value should on in the list SCOPE_LIST - if scope not in SCOPE_LIST: - raise LabellerrError(f"scope must be one of {', '.join(SCOPE_LIST)}") + if scope not in constants.SCOPE_LIST: + raise LabellerrError( + f"scope must be one of {', '.join(constants.SCOPE_LIST)}" + ) # get dataset try: unique_id = str(uuid.uuid4()) url = f"{self.base_url}/datasets/list?client_id={client_id}&data_type={datatype}&permission_level={scope}&project_id={project_id}&uuid={unique_id}" - headers = { - 'client_id': client_id, - 'content-type': 'application/json', - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://dev.labellerr.com' - } + headers = self._build_headers( + client_id=client_id, extra_headers={"content-type": "application/json"} + ) response = requests.request("GET", url, headers=headers) - - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) - return response.json() + return self._handle_response(response, unique_id) except LabellerrError as e: logging.error(f"Failed to retrieve dataset: {e}") raise - def get_total_folder_file_count_and_total_size(self,folder_path,data_type): + def get_total_folder_file_count_and_total_size(self, folder_path, data_type): """ - Retrieves the total count and size of files in a folder. + Retrieves the total count and size of files in a folder using memory-efficient iteration. :param folder_path: The path to the folder. :param data_type: The type of data for the files. :return: The total count and size of the files. """ - total_file_count=0 - total_file_size=0 - files_list=[] - for root, dirs, files in os.walk(folder_path): - for file in files: - file_path = os.path.join(root, file) - # print('>> ',file_path) - try: - # check if the file extention matching based on datatype - if not any(file.endswith(ext) for ext in DATA_TYPE_FILE_EXT[data_type]): - continue - files_list.append(file_path) - file_size = os.path.getsize(file_path) - total_file_count += 1 - total_file_size += file_size - except Exception as e: - print(f"Error reading {file_path}: {str(e)}") + total_file_count = 0 + total_file_size = 0 + files_list = [] + + # Use os.scandir for better performance and memory efficiency + def scan_directory(directory): + nonlocal total_file_count, total_file_size + try: + with os.scandir(directory) as entries: + for entry in entries: + if entry.is_file(): + file_path = entry.path + # Check if the file extension matches based on datatype + if not any( + file_path.endswith(ext) + for ext in constants.DATA_TYPE_FILE_EXT[data_type] + ): + continue + try: + file_size = entry.stat().st_size + files_list.append(file_path) + total_file_count += 1 + total_file_size += file_size + except OSError as e: + print(f"Error reading {file_path}: {str(e)}") + elif entry.is_dir(): + # Recursively scan subdirectories + scan_directory(entry.path) + except OSError as e: + print(f"Error scanning directory {directory}: {str(e)}") + scan_directory(folder_path) return total_file_count, total_file_size, files_list - - def get_total_file_count_and_total_size(self,files_list,data_type): + def get_total_file_count_and_total_size(self, files_list, data_type): """ Retrieves the total count and size of files in a list. @@ -383,15 +527,18 @@ def get_total_file_count_and_total_size(self,files_list,data_type): :param data_type: The type of data for the files. :return: The total count and size of the files. """ - total_file_count=0 - total_file_size=0 + total_file_count = 0 + total_file_size = 0 # for root, dirs, files in os.walk(folder_path): for file_path in files_list: if file_path is None: continue try: # check if the file extention matching based on datatype - if not any(file_path.endswith(ext) for ext in DATA_TYPE_FILE_EXT[data_type]): + if not any( + file_path.endswith(ext) + for ext in constants.DATA_TYPE_FILE_EXT[data_type] + ): continue file_size = os.path.getsize(file_path) total_file_count += 1 @@ -403,11 +550,7 @@ def get_total_file_count_and_total_size(self,files_list,data_type): return total_file_count, total_file_size, files_list - - - - def get_all_project_per_client_id(self,client_id): - + def get_all_project_per_client_id(self, client_id): """ Retrieves a list of projects associated with a client ID. @@ -419,37 +562,19 @@ def get_all_project_per_client_id(self,client_id): unique_id = str(uuid.uuid4()) url = f"{self.base_url}/project_drafts/projects/detailed_list?client_id={client_id}&uuid={unique_id}" - payload = {} - headers = { - 'client_id': str(client_id), - 'content-type': 'application/json', - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://dev.labellerr.com' - } - - response = requests.request("GET", url, headers=headers, data=payload) - - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) + headers = self._build_headers( + client_id=client_id, extra_headers={"content-type": "application/json"} + ) - # print(response.text) - return response.json() + response = requests.request("GET", url, headers=headers, data={}) + return self._handle_response(response, unique_id) except Exception as e: logging.error(f"Failed to retrieve projects: {str(e)}") raise LabellerrError(f"Failed to retrieve projects: {str(e)}") - - def create_annotation_guideline(self, client_id, questions, template_name, data_type): - + def create_annotation_guideline( + self, client_id, questions, template_name, data_type + ): """ Updates the annotation guideline for a project. @@ -461,50 +586,38 @@ def create_annotation_guideline(self, client_id, questions, template_name, data_ url = f"{constants.BASE_URL}/annotations/create_template?data_type={data_type}&client_id={client_id}&uuid={unique_id}" - guide_payload = json.dumps({ - "templateName": template_name, - "questions": questions - }) - - headers = { - 'client_id': str(client_id), - 'content-type': 'application/json', - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'source':'sdk', - 'origin': 'https://pro.labellerr.com' - } + guide_payload = json.dumps( + {"templateName": template_name, "questions": questions} + ) + + headers = self._build_headers( + client_id=client_id, extra_headers={"content-type": "application/json"} + ) try: - response = requests.request("POST", url, headers=headers, data=guide_payload) - - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) - rjson = response.json() - return rjson['response']['template_id'] + response = requests.request( + "POST", url, headers=headers, data=guide_payload + ) + response_data = self._handle_response(response, unique_id) + return response_data["response"]["template_id"] except requests.exceptions.RequestException as e: logging.error(f"Failed to update project annotation guideline: {str(e)}") - raise LabellerrError(f"Failed to update project annotation guideline: {str(e)}") - - + raise LabellerrError( + f"Failed to update project annotation guideline: {str(e)}" + ) - def validate_rotation_config(self,rotation_config): + def validate_rotation_config(self, rotation_config): """ Validates a rotation configuration. :param rotation_config: A dictionary containing the configuration for the rotations. :raises LabellerrError: If the configuration is invalid. """ - annotation_rotation_count = rotation_config.get('annotation_rotation_count') - review_rotation_count = rotation_config.get('review_rotation_count') - client_review_rotation_count = rotation_config.get('client_review_rotation_count') + annotation_rotation_count = rotation_config.get("annotation_rotation_count") + review_rotation_count = rotation_config.get("review_rotation_count") + client_review_rotation_count = rotation_config.get( + "client_review_rotation_count" + ) # Validate review_rotation_count if review_rotation_count != 1: @@ -512,14 +625,24 @@ def validate_rotation_config(self,rotation_config): # Validate client_review_rotation_count based on annotation_rotation_count if annotation_rotation_count == 0 and client_review_rotation_count != 0: - raise LabellerrError("client_review_rotation_count must be 0 when annotation_rotation_count is 0") - elif annotation_rotation_count == 1 and client_review_rotation_count not in [0, 1]: - raise LabellerrError("client_review_rotation_count can only be 0 or 1 when annotation_rotation_count is 1") + raise LabellerrError( + "client_review_rotation_count must be 0 when annotation_rotation_count is 0" + ) + elif annotation_rotation_count == 1 and client_review_rotation_count not in [ + 0, + 1, + ]: + raise LabellerrError( + "client_review_rotation_count can only be 0 or 1 when annotation_rotation_count is 1" + ) elif annotation_rotation_count > 1 and client_review_rotation_count != 0: - raise LabellerrError("client_review_rotation_count must be 0 when annotation_rotation_count is greater than 1") - + raise LabellerrError( + "client_review_rotation_count must be 0 when annotation_rotation_count is greater than 1" + ) - def _upload_preannotation_sync(self, project_id, client_id, annotation_format, annotation_file): + def _upload_preannotation_sync( + self, project_id, client_id, annotation_format, annotation_file + ): """ Synchronous implementation of preannotation upload. @@ -532,14 +655,21 @@ def _upload_preannotation_sync(self, project_id, client_id, annotation_format, a """ try: # validate all the parameters - required_params = ['project_id', 'client_id', 'annotation_format', 'annotation_file'] + required_params = [ + "project_id", + "client_id", + "annotation_format", + "annotation_file", + ] for param in required_params: if param not in locals(): raise LabellerrError(f"Required parameter {param} is missing") - - if annotation_format not in ANNOTATION_FORMAT: - raise LabellerrError(f"Invalid annotation_format. Must be one of {ANNOTATION_FORMAT}") - + + if annotation_format not in constants.ANNOTATION_FORMAT: + raise LabellerrError( + f"Invalid annotation_format. Must be one of {constants.ANNOTATION_FORMAT}" + ) + url = f"{self.base_url}/actions/upload_answers?project_id={project_id}&answer_format={annotation_format}&client_id={client_id}" # validate if the file exist then extract file name from the path @@ -549,13 +679,15 @@ def _upload_preannotation_sync(self, project_id, client_id, annotation_format, a raise LabellerrError("File not found") # Check if the file extension is .json when annotation_format is coco_json - if annotation_format == 'coco_json': + if annotation_format == "coco_json": file_extension = os.path.splitext(annotation_file)[1].lower() - if file_extension != '.json': - raise LabellerrError("For coco_json annotation format, the file must have a .json extension") + if file_extension != ".json": + raise LabellerrError( + "For coco_json annotation format, the file must have a .json extension" + ) # get the direct upload url gcs_path = f"{project_id}/{annotation_format}-{file_name}" - print ("Uploading your file to Labellerr. Please wait...") + print("Uploading your file to Labellerr. Please wait...") direct_upload_url = self.get_direct_upload_url(gcs_path, client_id) # Now let's wait for the file to be uploaded to the gcs gcs.upload_to_gcs_direct(direct_upload_url, annotation_file) @@ -568,50 +700,32 @@ def _upload_preannotation_sync(self, project_id, client_id, annotation_format, a # 'client_id': client_id, # 'api_key': self.api_key, # 'api_secret': self.api_secret, - # 'origin': 'https://dev.labellerr.com', + # 'origin': constants.ALLOWED_ORIGINS, # 'source':'sdk', # 'email_id': self.api_key # }, data=payload, files=files) url += "&gcs_path=" + gcs_path - response = requests.request("POST", url, headers={ - 'client_id': client_id, - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'origin': 'https://dev.labellerr.com', - 'source':'sdk', - 'email_id': self.api_key - }, data=payload) - response_data=response.json() - try: - response_data=response.json() - except Exception as e: - raise LabellerrError(f"Failed to upload preannotation: {response.text}") - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'error': response_data - }) + headers = self._build_headers( + client_id=client_id, extra_headers={"email_id": self.api_key} + ) + response = requests.request("POST", url, headers=headers, data=payload) + response_data = self._handle_upload_response(response) # read job_id from the response - job_id = response_data['response']['job_id'] + job_id = response_data["response"]["job_id"] self.client_id = client_id self.job_id = job_id self.project_id = project_id print(f"Preannotation upload successful. Job ID: {job_id}") - if response.status_code != 200: - raise LabellerrError(f"Failed to upload preannotation: {response.text}") - return self.preannotation_job_status() except Exception as e: logging.error(f"Failed to upload preannotation: {str(e)}") raise LabellerrError(f"Failed to upload preannotation: {str(e)}") - def upload_preannotation_by_project_id_async(self, project_id, client_id, annotation_format, annotation_file): + def upload_preannotation_by_project_id_async( + self, project_id, client_id, annotation_format, annotation_file + ): """ Asynchronously uploads preannotation data to a project. @@ -622,17 +736,25 @@ def upload_preannotation_by_project_id_async(self, project_id, client_id, annota :return: A Future object that will contain the response from the API. :raises LabellerrError: If the upload fails. """ + def upload_and_monitor(): try: # validate all the parameters - required_params = ['project_id', 'client_id', 'annotation_format', 'annotation_file'] + required_params = [ + "project_id", + "client_id", + "annotation_format", + "annotation_file", + ] for param in required_params: if param not in locals(): raise LabellerrError(f"Required parameter {param} is missing") - - if annotation_format not in ANNOTATION_FORMAT: - raise LabellerrError(f"Invalid annotation_format. Must be one of {ANNOTATION_FORMAT}") - + + if annotation_format not in constants.ANNOTATION_FORMAT: + raise LabellerrError( + f"Invalid annotation_format. Must be one of {constants.ANNOTATION_FORMAT}" + ) + url = f"{self.base_url}/actions/upload_answers?project_id={project_id}&answer_format={annotation_format}&client_id={client_id}" # validate if the file exist then extract file name from the path @@ -642,13 +764,15 @@ def upload_and_monitor(): raise LabellerrError("File not found") # Check if the file extension is .json when annotation_format is coco_json - if annotation_format == 'coco_json': + if annotation_format == "coco_json": file_extension = os.path.splitext(annotation_file)[1].lower() - if file_extension != '.json': - raise LabellerrError("For coco_json annotation format, the file must have a .json extension") + if file_extension != ".json": + raise LabellerrError( + "For coco_json annotation format, the file must have a .json extension" + ) # get the direct upload url gcs_path = f"{project_id}/{annotation_format}-{file_name}" - print ("Uploading your file to Labellerr. Please wait...") + print("Uploading your file to Labellerr. Please wait...") direct_upload_url = self.get_direct_upload_url(gcs_path, client_id) # Now let's wait for the file to be uploaded to the gcs gcs.upload_to_gcs_direct(direct_upload_url, annotation_file) @@ -661,68 +785,55 @@ def upload_and_monitor(): # 'client_id': client_id, # 'api_key': self.api_key, # 'api_secret': self.api_secret, - # 'origin': 'https://dev.labellerr.com', + # 'origin': constants.ALLOWED_ORIGINS, # 'source':'sdk', # 'email_id': self.api_key # }, data=payload, files=files) url += "&gcs_path=" + gcs_path - response = requests.request("POST", url, headers={ - 'client_id': client_id, - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'origin': 'https://dev.labellerr.com', - 'source':'sdk', - 'email_id': self.api_key - }, data=payload) - try: - response_data=response.json() - except Exception as e: - raise LabellerrError(f"Failed to upload preannotation: {response.text}") - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'error': response_data - }) + headers = self._build_headers( + client_id=client_id, extra_headers={"email_id": self.api_key} + ) + response = requests.request("POST", url, headers=headers, data=payload) + response_data = self._handle_upload_response(response) + # read job_id from the response - job_id = response_data['response']['job_id'] + job_id = response_data["response"]["job_id"] self.client_id = client_id self.job_id = job_id self.project_id = project_id print(f"Preannotation upload successful. Job ID: {job_id}") - if response.status_code != 200: - raise LabellerrError(f"Failed to upload preannotation: {response.text}") - + # Now monitor the status - headers = { - 'client_id': str(self.client_id), - 'Origin': 'https://app.labellerr.com', - 'api_key': self.api_key, - 'api_secret': self.api_secret - } + headers = self._build_headers( + client_id=self.client_id, + extra_headers={"Origin": constants.ALLOWED_ORIGINS}, + ) status_url = f"{self.base_url}/actions/upload_answers_status?project_id={self.project_id}&job_id={self.job_id}&client_id={self.client_id}" while True: try: - response = requests.request("GET", status_url, headers=headers, data={}) + response = requests.request( + "GET", status_url, headers=headers, data={} + ) status_data = response.json() - - print(' >>> ', status_data) + + print(" >>> ", status_data) # Check if job is completed - if status_data.get('response', {}).get('status') == 'completed': + if status_data.get("response", {}).get("status") == "completed": return status_data - - print('Syncing status after 5 seconds . . .') + + print("Syncing status after 5 seconds . . .") time.sleep(5) - + except Exception as e: - logging.error(f"Failed to get preannotation job status: {str(e)}") - raise LabellerrError(f"Failed to get preannotation job status: {str(e)}") - + logging.error( + f"Failed to get preannotation job status: {str(e)}" + ) + raise LabellerrError( + f"Failed to get preannotation job status: {str(e)}" + ) + except Exception as e: logging.exception(f"Failed to upload preannotation: {str(e)}") raise LabellerrError(f"Failed to upload preannotation: {str(e)}") @@ -733,40 +844,44 @@ def upload_and_monitor(): def preannotation_job_status_async(self): """ Get the status of a preannotation job asynchronously. - + Returns: concurrent.futures.Future: A future that will contain the final job status """ + def check_status(): - headers = { - 'client_id': str(self.client_id), - 'Origin': 'https://app.labellerr.com', - 'api_key': self.api_key, - 'api_secret': self.api_secret - } + headers = self._build_headers( + client_id=self.client_id, + extra_headers={"Origin": constants.ALLOWED_ORIGINS}, + ) url = f"{self.base_url}/actions/upload_answers_status?project_id={self.project_id}&job_id={self.job_id}&client_id={self.client_id}" payload = {} while True: try: - response = requests.request("GET", url, headers=headers, data=payload) + response = requests.request( + "GET", url, headers=headers, data=payload + ) response_data = response.json() - + # Check if job is completed - if response_data.get('response', {}).get('status') == 'completed': + if response_data.get("response", {}).get("status") == "completed": return response_data - - print('retrying after 5 seconds . . .') + + print("retrying after 5 seconds . . .") time.sleep(5) - + except Exception as e: logging.error(f"Failed to get preannotation job status: {str(e)}") - raise LabellerrError(f"Failed to get preannotation job status: {str(e)}") - + raise LabellerrError( + f"Failed to get preannotation job status: {str(e)}" + ) + with concurrent.futures.ThreadPoolExecutor() as executor: return executor.submit(check_status) - def upload_preannotation_by_project_id(self,project_id,client_id,annotation_format,annotation_file): - + def upload_preannotation_by_project_id( + self, project_id, client_id, annotation_format, annotation_file + ): """ Uploads preannotation data to a project. @@ -779,14 +894,20 @@ def upload_preannotation_by_project_id(self,project_id,client_id,annotation_form """ try: # validate all the parameters - required_params = ['project_id', 'client_id', 'annotation_format', 'annotation_file'] + required_params = [ + "project_id", + "client_id", + "annotation_format", + "annotation_file", + ] for param in required_params: if param not in locals(): raise LabellerrError(f"Required parameter {param} is missing") - - if annotation_format not in ANNOTATION_FORMAT: - raise LabellerrError(f"Invalid annotation_format. Must be one of {ANNOTATION_FORMAT}") - + + if annotation_format not in constants.ANNOTATION_FORMAT: + raise LabellerrError( + f"Invalid annotation_format. Must be one of {constants.ANNOTATION_FORMAT}" + ) url = f"{self.base_url}/actions/upload_answers?project_id={project_id}&answer_format={annotation_format}&client_id={client_id}" @@ -797,39 +918,39 @@ def upload_preannotation_by_project_id(self,project_id,client_id,annotation_form raise LabellerrError("File not found") payload = {} - with open(annotation_file, 'rb') as f: - files = [ - ('file', (file_name, f, 'application/octet-stream')) - ] - response = requests.request("POST", url, headers={ - 'client_id': client_id, - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'origin': 'https://dev.labellerr.com', - 'source':'sdk', - 'email_id': self.api_key - }, data=payload, files=files) - response_data=response.json() - print('response_data -- ', response_data) + with open(annotation_file, "rb") as f: + files = [("file", (file_name, f, "application/octet-stream"))] + headers = self._build_headers( + client_id=client_id, extra_headers={"email_id": self.api_key} + ) + response = requests.request( + "POST", url, headers=headers, data=payload, files=files + ) + response_data = self._handle_upload_response(response) + print("response_data -- ", response_data) + # read job_id from the response - job_id = response_data['response']['job_id'] + job_id = response_data["response"]["job_id"] self.client_id = client_id self.job_id = job_id self.project_id = project_id print(f"Preannotation upload successful. Job ID: {job_id}") - if response.status_code != 200: - raise LabellerrError(f"Failed to upload preannotation: {response.text}") - + future = self.preannotation_job_status_async() - return future.result() + return future.result() except Exception as e: logging.error(f"Failed to upload preannotation: {str(e)}") raise LabellerrError(f"Failed to upload preannotation: {str(e)}") - def create_local_export(self,project_id,client_id,export_config): + def create_local_export(self, project_id, client_id, export_config): unique_id = str(uuid.uuid4()) - required_params = ['export_name', 'export_description', 'export_format','statuses'] + required_params = [ + "export_name", + "export_description", + "export_format", + "statuses", + ] if project_id is None: raise LabellerrError("project_id cannot be null") @@ -843,61 +964,50 @@ def create_local_export(self,project_id,client_id,export_config): for param in required_params: if param not in export_config: raise LabellerrError(f"Required parameter {param} is missing") - if param == 'export_format': - if export_config[param] not in LOCAL_EXPORT_FORMAT: - raise LabellerrError(f"Invalid export_format. Must be one of {LOCAL_EXPORT_FORMAT}") - if param == 'statuses': + if param == "export_format": + if export_config[param] not in constants.LOCAL_EXPORT_FORMAT: + raise LabellerrError( + f"Invalid export_format. Must be one of {constants.LOCAL_EXPORT_FORMAT}" + ) + if param == "statuses": if not isinstance(export_config[param], list): - raise LabellerrError(f"Invalid statuses. Must be an array") + raise LabellerrError(f"Invalid statuses. Must be an array {param}") for status in export_config[param]: - if status not in LOCAL_EXPORT_STATUS: - raise LabellerrError(f"Invalid status. Must be one of {LOCAL_EXPORT_STATUS}") - + if status not in constants.LOCAL_EXPORT_STATUS: + raise LabellerrError( + f"Invalid status. Must be one of {constants.LOCAL_EXPORT_STATUS}" + ) try: - export_config.update({ - "export_destination": "local", - "question_ids": [ - "all" - ] - }) + export_config.update( + {"export_destination": "local", "question_ids": ["all"]} + ) payload = json.dumps(export_config) + headers = self._build_headers( + extra_headers={ + "Origin": constants.ALLOWED_ORIGINS, + "Content-Type": "application/json", + } + ) + response = requests.post( f"{self.base_url}/sdk/export/files?project_id={project_id}&client_id={client_id}", - headers={ - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'Origin': 'https://dev.labellerr.com', - 'Content-Type': 'application/json' - }, - data=payload + headers=headers, + data=payload, ) - if response.status_code not in [200, 201]: - if response.status_code >= 400 and response.status_code < 500: - raise LabellerrError({'error' :response.json(),'code':response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': unique_id - }) - return response.json() + return self._handle_response(response, unique_id) except requests.exceptions.RequestException as e: logging.error(f"Failed to create local export: {str(e)}") raise LabellerrError(f"Failed to create local export: {str(e)}") - - def fetch_download_url(self, api_key, api_secret, project_id, uuid, export_id, client_id): + def fetch_download_url( + self, api_key, api_secret, project_id, uuid, export_id, client_id + ): try: - headers = { - 'api_key': api_key, - 'api_secret': api_secret, - 'client_id': client_id, - 'origin': 'https://dev.labellerr.com', - 'source': 'sdk', - 'Content-Type': 'application/json' - } + headers = self._build_headers( + client_id=client_id, extra_headers={"Content-Type": "application/json"} + ) response = requests.get( url=f"{constants.BASE_URL}/exports/download", @@ -905,15 +1015,17 @@ def fetch_download_url(self, api_key, api_secret, project_id, uuid, export_id, c "client_id": client_id, "project_id": project_id, "uuid": uuid, - "report_id": export_id + "report_id": export_id, }, - headers=headers + headers=headers, ) if response.ok: return json.dumps(response.json().get("response"), indent=2) else: - raise LabellerrError(f" Download request failed: {response.status_code} - {response.text}") + raise LabellerrError( + f" Download request failed: {response.status_code} - {response.text}" + ) except requests.exceptions.RequestException as e: logging.error(f"Failed to download export: {str(e)}") raise LabellerrError(f"Failed to download export: {str(e)}") @@ -921,11 +1033,9 @@ def fetch_download_url(self, api_key, api_secret, project_id, uuid, export_id, c logging.error(f"Unexpected error in download_function: {str(e)}") raise LabellerrError(f"Unexpected error in download_function: {str(e)}") - - - - - def check_export_status(self, api_key, api_secret, project_id, report_ids, client_id): + def check_export_status( + self, api_key, api_secret, project_id, report_ids, client_id + ): request_uuid = str(uuid.uuid4()) try: if not project_id: @@ -937,44 +1047,31 @@ def check_export_status(self, api_key, api_secret, project_id, report_ids, clien url = f"{constants.BASE_URL}/exports/status?project_id={project_id}&uuid={request_uuid}&client_id={client_id}" # Headers - headers = { - 'client_id': client_id, - 'api_key': api_key, - 'api_secret': api_secret, - 'origin': 'https://dev.labellerr.com', - 'source': 'sdk', - 'Content-Type': 'application/json' - } + headers = self._build_headers( + client_id=client_id, extra_headers={"Content-Type": "application/json"} + ) - payload = json.dumps({ - "report_ids": report_ids - }) + payload = json.dumps({"report_ids": report_ids}) response = requests.post(url, headers=headers, data=payload) - - if response.status_code not in [200, 201]: - if 400 <= response.status_code < 500: - raise LabellerrError({'error': response.json(), 'code': response.status_code}) - elif response.status_code >= 500: - raise LabellerrError({ - 'status': 'Internal server error', - 'message': 'Please contact support with the request tracking id', - 'request_id': request_uuid - }) - - result = response.json() + result = self._handle_response(response, request_uuid) # Now process each report_id for status_item in result.get("status", []): - if status_item.get("is_completed") and status_item.get("export_status") == "Created": + if ( + status_item.get("is_completed") + and status_item.get("export_status") == "Created" + ): # Download URL if job completed - download_url = self.fetch_download_url( - api_key=api_key, - api_secret=api_secret, - project_id=project_id, - uuid=request_uuid, - export_id=status_item["report_id"], - client_id=client_id + download_url = ( # noqa E999 todo check use of that + self.fetch_download_url( + api_key=api_key, + api_secret=api_secret, + project_id=project_id, + uuid=request_uuid, + export_id=status_item["report_id"], + client_id=client_id, + ) ) return json.dumps(result, indent=2) @@ -986,162 +1083,184 @@ def check_export_status(self, api_key, api_secret, project_id, report_ids, clien logging.error(f"Unexpected error checking export status: {str(e)}") raise LabellerrError(f"Unexpected error checking export status: {str(e)}") - - def create_project(self, project_name, data_type, client_id, dataset_id, annotation_template_id, rotation_config, created_by=None): + def create_project( + self, + project_name, + data_type, + client_id, + dataset_id, + annotation_template_id, + rotation_config, + created_by=None, + ): """ Creates a project with the given configuration. """ url = f"{constants.BASE_URL}/projects/create?client_id={client_id}" - - - payload = json.dumps({ - "project_name": project_name, - "attached_datasets": [dataset_id], - "data_type": data_type, - "annotation_template_id": annotation_template_id, - "rotations": rotation_config, - "created_by": created_by - }) - - headers = { - 'api_key': self.api_key, - 'api_secret': self.api_secret, - 'Origin': 'https://pro.labellerr.com', - 'Content-Type': 'application/json' - } - + + payload = json.dumps( + { + "project_name": project_name, + "attached_datasets": [dataset_id], + "data_type": data_type, + "annotation_template_id": annotation_template_id, + "rotations": rotation_config, + "created_by": created_by, + } + ) + + headers = self._build_headers( + extra_headers={ + "Origin": constants.ALLOWED_ORIGINS, + "Content-Type": "application/json", + } + ) + # print(f"{payload}") - + response = requests.post(url, headers=headers, data=payload) response_data = response.json() - - # print(f"{response_data}") - - - if 'error' in response_data and response_data['error']: - error_details = response_data['error'] - error_msg = f"Validation Error: {response_data.get('message', 'Unknown error')}" + + # print(f"{response_data}") + + if "error" in response_data and response_data["error"]: + error_details = response_data["error"] + error_msg = ( + f"Validation Error: {response_data.get('message', 'Unknown error')}" + ) for error in error_details: error_msg += f"\n- Field '{error['field']}': {error['message']}" raise LabellerrError(error_msg) - - return response_data + return response_data def initiate_create_project(self, payload): """ Orchestrates project creation by handling dataset creation, annotation guidelines, and final project setup. """ - + try: - result = {} # validate all the parameters - required_params = ['client_id', 'dataset_name', 'dataset_description', 'data_type', 'created_by', 'project_name','annotation_guide','autolabel'] + required_params = [ + "client_id", + "dataset_name", + "dataset_description", + "data_type", + "created_by", + "project_name", + "annotation_guide", + "autolabel", + ] for param in required_params: if param not in payload: raise LabellerrError(f"Required parameter {param} is missing") - - - if param == 'client_id' and not isinstance(payload[param], str): + + if param == "client_id" and not isinstance(payload[param], str): raise LabellerrError("client_id must be a non-empty string") - - if param == 'annotation_guide': - for guide in payload['annotation_guide']: - if 'option_type' not in guide: - raise LabellerrError("option_type is required in annotation_guide") - if guide['option_type'] not in OPTION_TYPE_LIST: - raise LabellerrError(f"option_type must be one of {OPTION_TYPE_LIST}") - - - if 'folder_to_upload' in payload and 'files_to_upload' in payload: - raise LabellerrError("Cannot provide both files_to_upload and folder_to_upload") - - if 'folder_to_upload' not in payload and 'files_to_upload' not in payload: - raise LabellerrError("Either files_to_upload or folder_to_upload must be provided") - - - if 'rotation_config' not in payload: - payload['rotation_config'] = { - 'annotation_rotation_count': 1, - 'review_rotation_count': 1, - 'client_review_rotation_count': 1 + + if param == "annotation_guide": + for guide in payload["annotation_guide"]: + if "option_type" not in guide: + raise LabellerrError( + "option_type is required in annotation_guide" + ) + if guide["option_type"] not in constants.OPTION_TYPE_LIST: + raise LabellerrError( + f"option_type must be one of {constants.OPTION_TYPE_LIST}" + ) + + if "folder_to_upload" in payload and "files_to_upload" in payload: + raise LabellerrError( + "Cannot provide both files_to_upload and folder_to_upload" + ) + + if "folder_to_upload" not in payload and "files_to_upload" not in payload: + raise LabellerrError( + "Either files_to_upload or folder_to_upload must be provided" + ) + + if "rotation_config" not in payload: + payload["rotation_config"] = { + "annotation_rotation_count": 1, + "review_rotation_count": 1, + "client_review_rotation_count": 1, } - self.validate_rotation_config(payload['rotation_config']) - - - if payload['data_type'] not in DATA_TYPES: - raise LabellerrError(f"Invalid data_type. Must be one of {DATA_TYPES}") - + self.validate_rotation_config(payload["rotation_config"]) + + if payload["data_type"] not in constants.DATA_TYPES: + raise LabellerrError( + f"Invalid data_type. Must be one of {constants.DATA_TYPES}" + ) + print("Rotation configuration validated . . .") - - + print("Creating dataset . . .") - dataset_response = self.create_dataset({ - 'client_id': payload['client_id'], - 'dataset_name': payload['dataset_name'], - 'data_type': payload['data_type'], - 'dataset_description': payload['dataset_description'], - }, - files_to_upload=payload.get('files_to_upload'), - folder_to_upload=payload.get('folder_to_upload')) - - dataset_id = dataset_response['dataset_id'] - - + dataset_response = self.create_dataset( + { + "client_id": payload["client_id"], + "dataset_name": payload["dataset_name"], + "data_type": payload["data_type"], + "dataset_description": payload["dataset_description"], + }, + files_to_upload=payload.get("files_to_upload"), + folder_to_upload=payload.get("folder_to_upload"), + ) + + dataset_id = dataset_response["dataset_id"] + def dataset_ready(): try: - dataset_status = self.get_dataset(payload['client_id'], dataset_id) - + dataset_status = self.get_dataset(payload["client_id"], dataset_id) + if isinstance(dataset_status, dict): - - if 'response' in dataset_status: - return dataset_status['response'].get('status_code', 200) == 300 + + if "response" in dataset_status: + return ( + dataset_status["response"].get("status_code", 200) + == 300 + ) else: - + return True return False except Exception as e: print(f"Error checking dataset status: {e}") return False - - + utils.poll( function=dataset_ready, condition=lambda x: x is True, interval=5, - timeout=60 + timeout=60, ) - + print("Dataset created and ready for use") - - + annotation_template_id = self.create_annotation_guideline( - payload['client_id'], - payload['annotation_guide'], - payload['project_name'], - payload['data_type'] + payload["client_id"], + payload["annotation_guide"], + payload["project_name"], + payload["data_type"], ) print("Annotation guidelines created") - - + project_response = self.create_project( - project_name=payload['project_name'], - data_type=payload['data_type'], - client_id=payload['client_id'], + project_name=payload["project_name"], + data_type=payload["data_type"], + client_id=payload["client_id"], dataset_id=dataset_id, annotation_template_id=annotation_template_id, - rotation_config=payload['rotation_config'], - created_by=payload['created_by'] + rotation_config=payload["rotation_config"], + created_by=payload["created_by"], ) - - + return { - 'status': 'success', - 'message': 'Project created successfully', - 'project_id': project_response + "status": "success", + "message": "Project created successfully", + "project_id": project_response, } - + except LabellerrError as e: logging.error(f"Project creation failed: {str(e)}") raise @@ -1159,82 +1278,110 @@ def upload_folder_files_to_dataset(self, data_config): """ try: # Validate required fields in data_config - required_fields = [ 'client_id', 'folder_path', 'data_type'] - missing_fields = [field for field in required_fields if field not in data_config] + required_fields = ["client_id", "folder_path", "data_type"] + missing_fields = [ + field for field in required_fields if field not in data_config + ] if missing_fields: - raise LabellerrError(f"Missing required fields in data_config: {', '.join(missing_fields)}") + raise LabellerrError( + f"Missing required fields in data_config: {', '.join(missing_fields)}" + ) # Validate folder path exists and is accessible - if not os.path.exists(data_config['folder_path']): - raise LabellerrError(f"Folder path does not exist: {data_config['folder_path']}") - if not os.path.isdir(data_config['folder_path']): - raise LabellerrError(f"Path is not a directory: {data_config['folder_path']}") - if not os.access(data_config['folder_path'], os.R_OK): - raise LabellerrError(f"No read permission for folder: {data_config['folder_path']}") + if not os.path.exists(data_config["folder_path"]): + raise LabellerrError( + f"Folder path does not exist: {data_config['folder_path']}" + ) + if not os.path.isdir(data_config["folder_path"]): + raise LabellerrError( + f"Path is not a directory: {data_config['folder_path']}" + ) + if not os.access(data_config["folder_path"], os.R_OK): + raise LabellerrError( + f"No read permission for folder: {data_config['folder_path']}" + ) success_queue = [] fail_queue = [] try: # Get files from folder - total_file_count, total_file_volumn, filenames = self.get_total_folder_file_count_and_total_size( - data_config['folder_path'], - data_config['data_type'] + total_file_count, total_file_volumn, filenames = ( + self.get_total_folder_file_count_and_total_size( + data_config["folder_path"], data_config["data_type"] + ) ) except Exception as e: raise LabellerrError(f"Failed to analyze folder contents: {str(e)}") - + # Check file limits - if total_file_count > TOTAL_FILES_COUNT_LIMIT_PER_DATASET: - raise LabellerrError(f"Total file count: {total_file_count} exceeds limit of {TOTAL_FILES_COUNT_LIMIT_PER_DATASET} files") - if total_file_volumn > TOTAL_FILES_SIZE_LIMIT_PER_DATASET: - raise LabellerrError(f"Total file size: {total_file_volumn/1024/1024:.1f}MB exceeds limit of {TOTAL_FILES_SIZE_LIMIT_PER_DATASET/1024/1024:.1f}MB") + if total_file_count > constants.TOTAL_FILES_COUNT_LIMIT_PER_DATASET: + raise LabellerrError( + f"Total file count: {total_file_count} exceeds limit of {constants.TOTAL_FILES_COUNT_LIMIT_PER_DATASET} files" + ) + if total_file_volumn > constants.TOTAL_FILES_SIZE_LIMIT_PER_DATASET: + raise LabellerrError( + f"Total file size: {total_file_volumn/1024/1024:.1f}MB exceeds limit of {constants.TOTAL_FILES_SIZE_LIMIT_PER_DATASET/1024/1024:.1f}MB" + ) print(f"Total file count: {total_file_count}") print(f"Total file size: {total_file_volumn/1024/1024:.1f} MB") - # Group files into batches based on FILE_BATCH_SIZE - batches = [] - current_batch = [] - current_batch_size = 0 + # Use generator for memory-efficient batch creation + def create_batches(): + current_batch = [] + current_batch_size = 0 - for file_path in filenames: - try: - file_size = os.path.getsize(file_path) - if current_batch_size + file_size > FILE_BATCH_SIZE or len(current_batch) >= FILE_BATCH_COUNT: - if current_batch: - batches.append(current_batch) - current_batch = [file_path] - current_batch_size = file_size - else: - current_batch.append(file_path) - current_batch_size += file_size - except OSError as e: - print(f"Error accessing file {file_path}: {str(e)}") - fail_queue.append(file_path) - except Exception as e: - print(f"Unexpected error processing {file_path}: {str(e)}") - fail_queue.append(file_path) + for file_path in filenames: + try: + file_size = os.path.getsize(file_path) + if ( + current_batch_size + file_size > constants.FILE_BATCH_SIZE + or len(current_batch) >= constants.FILE_BATCH_COUNT + ): + if current_batch: + yield current_batch + current_batch = [file_path] + current_batch_size = file_size + else: + current_batch.append(file_path) + current_batch_size += file_size + except OSError as e: + print(f"Error accessing file {file_path}: {str(e)}") + fail_queue.append(file_path) + except Exception as e: + print(f"Unexpected error processing {file_path}: {str(e)}") + fail_queue.append(file_path) - if current_batch: - batches.append(current_batch) + if current_batch: + yield current_batch + + # Convert generator to list for ThreadPoolExecutor + batches = list(create_batches()) if not batches: - raise LabellerrError("No valid files found to upload in the specified folder") + raise LabellerrError( + "No valid files found to upload in the specified folder" + ) - print('CPU count', cpu_count(), " Batch Count", len(batches)) + print("CPU count", cpu_count(), " Batch Count", len(batches)) # Calculate optimal number of workers based on CPU count and batch count max_workers = min( cpu_count(), # Number of CPU cores len(batches), # Number of batches - 20 + 20, ) connection_id = str(uuid.uuid4()) # Process batches in parallel with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_batch = { - executor.submit(self.__process_batch, data_config['client_id'], batch, connection_id): batch + executor.submit( + self.__process_batch, + data_config["client_id"], + batch, + connection_id, + ): batch for batch in batches } @@ -1242,7 +1389,10 @@ def upload_folder_files_to_dataset(self, data_config): batch = future_to_batch[future] try: result = future.result() - if result['message'] == '200: Success': + if ( + isinstance(result, dict) + and result.get("message") == "200: Success" + ): success_queue.extend(batch) else: fail_queue.extend(batch) @@ -1252,16 +1402,17 @@ def upload_folder_files_to_dataset(self, data_config): fail_queue.extend(batch) if not success_queue and fail_queue: - raise LabellerrError("All file uploads failed. Check individual file errors above.") + raise LabellerrError( + "All file uploads failed. Check individual file errors above." + ) return { - 'connection_id': connection_id, - 'success': success_queue, - 'fail': fail_queue + "connection_id": connection_id, + "success": success_queue, + "fail": fail_queue, } - + except LabellerrError as e: raise e except Exception as e: raise LabellerrError(f"Failed to upload files: {str(e)}") - diff --git a/labellerr/config.py b/labellerr/config.py index 5bdc0c4..53b1e4b 100644 --- a/labellerr/config.py +++ b/labellerr/config.py @@ -1 +1 @@ -cdn_server_address = 'cdn-951134552678.us-central1.run.app:443' \ No newline at end of file +cdn_server_address = "cdn-951134552678.us-central1.run.app:443" diff --git a/labellerr/constants.py b/labellerr/constants.py index 4bf50c2..d0a89ce 100644 --- a/labellerr/constants.py +++ b/labellerr/constants.py @@ -1 +1,43 @@ -BASE_URL = "https://api.labellerr.com" \ No newline at end of file +BASE_URL = "https://api.labellerr.com" +ALLOWED_ORIGINS = "https://pro.labellerr.com" + + +FILE_BATCH_SIZE = 15 * 1024 * 1024 # 15MB +FILE_BATCH_COUNT = 900 +TOTAL_FILES_SIZE_LIMIT_PER_DATASET = 2.5 * 1024 * 1024 * 1024 # 2.5GB +TOTAL_FILES_COUNT_LIMIT_PER_DATASET = 2500 + +ANNOTATION_FORMAT = ["json", "coco_json", "csv", "png"] +LOCAL_EXPORT_FORMAT = ["json", "coco_json", "csv", "png"] +LOCAL_EXPORT_STATUS = [ + "review", + "r_assigned", + "client_review", + "cr_assigned", + "accepted", +] + +# DATA TYPES: image, video, audio, document, text +DATA_TYPES = ("image", "video", "audio", "document", "text") +DATA_TYPE_FILE_EXT = { + "image": [".jpg", ".jpeg", ".png", ".tiff"], + "video": [".mp4"], + "audio": [".mp3", ".wav"], + "document": [".pdf"], + "text": [".txt"], +} + +SCOPE_LIST = ["project", "client", "public"] +OPTION_TYPE_LIST = [ + "input", + "radio", + "boolean", + "select", + "dropdown", + "stt", + "imc", + "BoundingBox", + "polygon", + "dot", + "audio", +] diff --git a/labellerr/exceptions.py b/labellerr/exceptions.py index 404aef6..b8902ed 100644 --- a/labellerr/exceptions.py +++ b/labellerr/exceptions.py @@ -1,6 +1,7 @@ # labellerr/exceptions.py + class LabellerrError(Exception): """Custom exception for Labellerr SDK errors.""" - pass + pass diff --git a/labellerr/gcs.py b/labellerr/gcs.py index 8c845e5..a7f16ae 100644 --- a/labellerr/gcs.py +++ b/labellerr/gcs.py @@ -1,47 +1,92 @@ import os + import requests -CONTENT_TYPE = 'application/octet-stream' -def upload_to_gcs_direct(signed_url, file_path): - with open(file_path, "rb") as f: - file_data = f.read() - headers = { - "Content-Type": CONTENT_TYPE, - } - upload_response = requests.put(signed_url, headers=headers, data=file_data) +from .exceptions import LabellerrError + +CONTENT_TYPE = "application/octet-stream" + - if upload_response.status_code in (200, 201): +def _handle_gcs_response(response, operation_name="GCS operation"): + """ + Standardized error handling for Google Cloud Storage operations. + + :param response: requests.Response object + :param operation_name: Name of the operation for error messages + :return: True for successful operations + :raises LabellerrError: For non-successful responses + """ + if operation_name == "resumable_start": + expected_codes = [201] + elif operation_name == "upload": + expected_codes = [200, 201] + else: + expected_codes = [200] + + if response.status_code in expected_codes: return True else: - raise AssertionError(f"Upload failed: {upload_response.status_code}, {upload_response.text}") -def upload_to_gcs_resumable(signed_url, file_path): + raise LabellerrError( + f"{operation_name} failed: {response.status_code} - {response.text}" + ) + + +def upload_to_gcs_direct(signed_url, file_path, chunk_size=8192): + """ + Upload file to GCS using streaming to minimize memory usage. + + :param signed_url: GCS signed URL for upload + :param file_path: Local file path to upload + :param chunk_size: Size of chunks to read (default 8KB) + """ + file_size = os.path.getsize(file_path) + headers = {"Content-Type": CONTENT_TYPE, "Content-Length": str(file_size)} + # Use streaming upload to minimize memory usage + with open(file_path, "rb") as f: + upload_response = requests.put(signed_url, headers=headers, data=f) + + _handle_gcs_response(upload_response, "direct upload") + return True + + +def upload_to_gcs_resumable(signed_url, file_path, chunk_size=1024 * 1024): + """ + Upload file to GCS using resumable upload with streaming for memory efficiency. + + :param signed_url: GCS signed URL for resumable upload + :param file_path: Local file path to upload + :param chunk_size: Size of chunks to upload (default 1MB) + """ # Step 1: Start a resumable upload session + file_size = os.path.getsize(file_path) headers = { "x-goog-resumable": "start", "Content-Type": CONTENT_TYPE, + "Content-Length": "0", } response = requests.post(signed_url, headers=headers) - - if response.status_code != 201: - raise AssertionError(f"Failed to start resumable session: {response.status_code}, {response.text}") - + _handle_gcs_response(response, "resumable_start") upload_url = response.headers["Location"] - # Step 2: Upload the whole file in a single PUT request - file_size = os.path.getsize(file_path) + # Step 2: Upload the file in chunks using streaming with open(file_path, "rb") as f: - file_data = f.read() + if file_size <= chunk_size: + # Small file - upload in one chunk + headers = { + "Content-Type": CONTENT_TYPE, + "Content-Range": f"bytes 0-{file_size-1}/{file_size}", + "Content-Length": str(file_size), + } + upload_response = requests.put(upload_url, headers=headers, data=f) + else: + # Large file - upload using streaming + headers = { + "Content-Type": CONTENT_TYPE, + "Content-Range": f"bytes 0-{file_size-1}/{file_size}", + "Content-Length": str(file_size), + } + upload_response = requests.put(upload_url, headers=headers, data=f) - headers = { - "Content-Type": CONTENT_TYPE, - "Content-Range": f"bytes 0-{file_size-1}/{file_size}", - } - - upload_response = requests.put(upload_url, headers=headers, data=file_data) - - if upload_response.status_code in (200, 201): - return True - else: - raise AssertionError(f"Upload failed: {upload_response.status_code}, {upload_response.text}") - \ No newline at end of file + _handle_gcs_response(upload_response, "resumable upload") + return True diff --git a/labellerr/utils.py b/labellerr/utils.py index 04a5f22..d7e3431 100644 --- a/labellerr/utils.py +++ b/labellerr/utils.py @@ -1,9 +1,9 @@ -import time import logging -from typing import Callable, Any, Optional, TypeVar, Union -import requests +import time +from typing import Any, Callable, Optional, TypeVar, Union + +T = TypeVar("T") -T = TypeVar('T') def poll( function: Callable[..., T], @@ -15,11 +15,11 @@ def poll( kwargs: dict = None, on_success: Optional[Callable[[T], Any]] = None, on_timeout: Optional[Callable[[int, Optional[T]], Any]] = None, - on_exception: Optional[Callable[[Exception], Any]] = None + on_exception: Optional[Callable[[Exception], Any]] = None, ) -> Union[T, None]: """ Poll a function at specified intervals until a condition is met. - + Args: function: The function to call condition: Function that takes the return value of `function` and returns True when polling should stop @@ -31,10 +31,10 @@ def poll( on_success: Callback function to call with the successful result on_timeout: Callback function to call on timeout with the number of attempts and last result on_exception: Callback function to call when an exception occurs in `function` - + Returns: The last return value from `function` or None if timeout/max_retries was reached - + Examples: ```python # Poll until a job is complete @@ -45,7 +45,7 @@ def poll( timeout=300, args=(job_id,) ) - + # Poll with a custom breaking condition result = poll( function=get_task_result, @@ -57,40 +57,42 @@ def poll( """ if kwargs is None: kwargs = {} - + start_time = time.time() attempts = 0 last_result = None - + while True: try: attempts += 1 last_result = function(*args, **kwargs) - + # Check if condition is satisfied if condition(last_result): if on_success: on_success(last_result) return last_result - + except Exception as e: if on_exception: on_exception(e) logging.error(f"Exception in poll function: {str(e)}") - + # Check if we've reached timeout if timeout is not None and time.time() - start_time > timeout: if on_timeout: on_timeout(attempts, last_result) - logging.warning(f"Polling timed out after {timeout} seconds ({attempts} attempts)") + logging.warning( + f"Polling timed out after {timeout} seconds ({attempts} attempts)" + ) return last_result - + # Check if we've reached max retries if max_retries is not None and attempts >= max_retries: if on_timeout: on_timeout(attempts, last_result) logging.warning(f"Polling reached max retries: {max_retries}") return last_result - + # Wait before next attempt time.sleep(interval) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..da5f0b5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,127 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "labellerr-sdk" +version = "1.0.0" +description = "Python SDK for Labellerr API" +readme = "README.md" +license = {text = "MIT"} +authors = [ + {name = "Labellerr Team", email = "support@labellerr.com"} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +keywords = ["labellerr", "api", "sdk", "machine-learning", "data-annotation", "async"] +requires-python = ">=3.7" +dependencies = [ + "requests>=2.25.0", + "urllib3>=1.26.0", + "aiohttp>=3.8.0", + "aiofiles>=0.8.0", + "certifi>=2021.5.25", +] + +[project.optional-dependencies] +dev = [ + "pytest>=6.0", + "pytest-asyncio>=0.18.0", + "pytest-cov>=2.10.0", + "black>=21.0.0", + "flake8>=3.8.0", + "mypy>=0.800", + "isort>=5.0.0", + "build>=0.3.0", +] +docs = [ + "sphinx>=4.0.0", + "sphinx-rtd-theme>=0.5.0", + "myst-parser>=0.15.0", +] + +[project.urls] +Homepage = "https://github.com/tensormatics/SDKPython" +Documentation = "https://github.com/tensormatics/SDKPython#readme" +Repository = "https://github.com/tensormatics/SDKPython" +"Bug Tracker" = "https://github.com/tensormatics/SDKPython/issues" +Changelog = "https://github.com/tensormatics/SDKPython/blob/main/CHANGELOG.md" + +[tool.setuptools.packages.find] +include = ["labellerr*"] +exclude = ["tests*", "docs*"] + +[tool.setuptools.package-data] +labellerr = ["py.typed"] + +[tool.black] +line-length = 88 +target-version = ['py37'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +line_length = 88 +known_first_party = ["labellerr"] + +[tool.mypy] +python_version = "3.7" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q --strict-markers" +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] + +[tool.coverage.run] +source = ["labellerr"] +omit = ["tests/*", "*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3288e92..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests - diff --git a/setup.py b/setup.py deleted file mode 100644 index f3ecdad..0000000 --- a/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# setup.py - -from setuptools import setup, find_packages - -setup( - name="labellerr_sdk", - version="1.0.0", - packages=find_packages(), - install_requires=[ - "requests", - "unique_names_generator" - ], - description="Python SDK for Labellerr API", - author="Your Name", - author_email="your.email@example.com", - url="https://github.com/tensormatics/SDKPython", - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - python_requires='>=3.6', -) - diff --git a/tests/__pycache__/__init__.cpython-310.pyc b/tests/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 1b1c8d7e4ced92b29e6b0157ffd6327ad9baa53b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 155 zcmd1j<>g`kf>Yi6=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vGKeRZts8~NS zFFi4@I5kh-B|o_|H#M)MSU)E*DK#gxs7OE9#XF#~BqKjhza+I7C=nl@nU`4-AFo$X Xd5gm)H$SB`C)EyQZZQ*(U||3N=TRg+ diff --git a/tests/__pycache__/test_client.cpython-310.pyc b/tests/__pycache__/test_client.cpython-310.pyc deleted file mode 100644 index 610757eb79424c4945a59b2e4e7f63914f3a5830..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1542 zcma)6&2Hm15GEzsmYv`3?xjErs0DiP#s6{aqCY8;pobPg0dIQXMW98>t1KnDq`cTQ z9ORHa_7V1w9Q!hT09|`3`UXXTc1S0w)Ampb9L|h5!{IkGR69Ej1Y_&BceCZ^2>oJ( z%|Spof@xj>;fQ02s9imYGmm=M^1k#lLW$#vtYlSMbv%|eT5~o54cuC%b+n# zy!saLsz4W8*FN1AJG}PN(_?{dc5e}Qz!Nu{!LM*OI&0#x8p~AV1#o-cvZ;_#XnlOI z+I>)+YOQo^jW&k|!VygK0T6*G=7@S6bMGypKKD6+mGH_0HLK;*=fV^Z{?Yl&V*B-24Z9`jx-csLz!5sweXQ`U>)QGeX&Gk)0VAN2d(!C)xj1Hr|37_**e zdfLXJ?d<>;VuVb7Lb?Xx?_aggjnJme@)^sG$lE6>USt5EX-f+qpx2(9eDmF#;)TlF z1zg!U8m(b&&EJ$2BZ~96y`Zrt?0d`i{{xzq+w};b3hcJi0dxWk$Cm)mrDu`0sh?q9 zdq*z)6}mt-c!gKq%D?q5$qL)|=&;-Eb|@JQM!n^hi}LjKT<2p+OLR|P9lm79`W z7?soD{$G>OE#a)AoMnPOxf8fBu}11-&B(8?&(^v~uh{)}V)^5uNGf+vzbG;B{Wv)}`!HTXzR%v6d z)rE+QDCM-it{cnV#}lhONoQqM3j?`vWw|B+7?p_ zo3_Pke;O+VD8T4y`%9;_&NG(gQPkX{!6Hu!8v?Dnp0Z7~26F#FZmn@~jY^;Wtro9s z@@>kseP^|OiYZwvER@9yNP{ivk`!qs%D`Uz7;4vrplwH%#740Zx4VV|Hu!9P=P&F# Z?wK?9o_o%j zbAG>je`s!QL@++SerWpJ284cNFaD4=mG$>QSwa|LE{D?W$>+E{pXNEH3ppV#rbSQZ za#C9I?Buk}qh}BnuOTcEG}Gcctv^?Us-6xI1yI&%NI06ru@TcfX^!GVBpy#jW2P}OHIm50?&x{O* zBVioNkOYY(lJT%P9EnVUh$b?`#6~<8Hl~J&K{TGS%v{;@>mu4Dm$6wYJsYXed7Dr> zWLVRNWfLnjRxpcssGl9mv4TQ9Lobd!cW%yktzd;5D1+@uz3OdfcFvV-;+&tI@7ke8 zL*7yVBH3LauA=*$y?6V@D+oQ$P4Mdo?V03D;s;HjA+)8SGE8^{bwL5haf?vMMIH-_ z0u~oV1&TStHfViBZF ziCxS&E=O!u<(}%GP0U3zLz3=CoIkh|cD9(XUG^mqS5dhmTxu-0cbAm%?jxl@xiwxA zE4-)-mOHwZE`D_JUWc~Yp)L1+oBeL@_pzTk&XvyGSKB^0ywv|u|IL?|Q)|JYpViP` z_gfDj%R zpzjRU7XTvL!HOj;V)=97|AF>aLmjpN`5JH`v(1463z82}j?tn9Y2N}NhjrK+Gd0-y zGjJ|RIKb$Mm0OLzr#g6@X5Z3j^|hH@zE*#N7v)yd?hjql=<=wXIx*e(_vU+K^*Qd!FCYfc(t{%o!ICvNdu=brc0770WAr?hsF=zzT zdZrtJr>b9ff{wxh Date: Mon, 21 Jul 2025 10:53:07 +0530 Subject: [PATCH 2/6] fix ci and code review comment --- .github/workflows/ci.yml | 2 +- labellerr/async_client.py | 23 ++--- labellerr/client.py | 187 +++++++++++--------------------------- labellerr/client_utils.py | 169 ++++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 149 deletions(-) create mode 100644 labellerr/client_utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6bf56b..5a8896f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: jobs: test: name: Test Suite - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] diff --git a/labellerr/async_client.py b/labellerr/async_client.py index e4cd26d..9afb8d7 100644 --- a/labellerr/async_client.py +++ b/labellerr/async_client.py @@ -9,7 +9,7 @@ import aiofiles import aiohttp -from . import constants +from . import constants, client_utils from .exceptions import LabellerrError @@ -74,20 +74,13 @@ def _build_headers( :param extra_headers: Optional dictionary of additional headers :return: Dictionary of headers """ - headers = { - "api_key": self.api_key, - "api_secret": self.api_secret, - "source": "sdk-async", - "origin": constants.ALLOWED_ORIGINS, - } - - if client_id: - headers["client_id"] = str(client_id) - - if extra_headers: - headers.update(extra_headers) - - return headers + return client_utils.build_headers( + api_key=self.api_key, + api_secret=self.api_secret, + source="sdk-async", + client_id=client_id, + extra_headers=extra_headers + ) async def _handle_response( self, response: aiohttp.ClientResponse, request_id: Optional[str] = None diff --git a/labellerr/client.py b/labellerr/client.py index f044015..a9b4480 100644 --- a/labellerr/client.py +++ b/labellerr/client.py @@ -10,8 +10,9 @@ from multiprocessing import cpu_count import requests - -from . import constants, gcs, utils +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from . import constants, gcs, utils, client_utils from .exceptions import LabellerrError # python -m unittest discover -s tests --run @@ -56,14 +57,6 @@ def _setup_session(self): """ Set up requests session with connection pooling for better performance. """ - try: - from requests.adapters import HTTPAdapter - from urllib3.util.retry import Retry - except ImportError: - # Fallback if urllib3 is not available - HTTPAdapter = None - Retry = None - self._session = requests.Session() if HTTPAdapter and Retry: @@ -129,20 +122,13 @@ def _build_headers(self, client_id=None, extra_headers=None): :param extra_headers: Optional dictionary of additional headers :return: Dictionary of headers """ - headers = { - "api_key": self.api_key, - "api_secret": self.api_secret, - "source": "sdk", - "origin": constants.ALLOWED_ORIGINS, - } - - if client_id: - headers["client_id"] = str(client_id) - - if extra_headers: - headers.update(extra_headers) - - return headers + return client_utils.build_headers( + api_key=self.api_key, + api_secret=self.api_secret, + source="sdk", + client_id=client_id, + extra_headers=extra_headers + ) def _handle_response(self, response, request_id=None, success_codes=None): """ @@ -360,11 +346,11 @@ def update_rotation_count(self): ) payload = json.dumps(self.rotation_config) - print(f"Update Rotation Count Payload: {payload}") + logging.info(f"Update Rotation Count Payload: {payload}") response = requests.request("POST", url, headers=headers, data=payload) - print("Rotation configuration updated successfully.") + logging.info("Rotation configuration updated successfully.") self._handle_response(response, unique_id) return {"msg": "project rotation configuration updated"} @@ -509,12 +495,12 @@ def scan_directory(directory): total_file_count += 1 total_file_size += file_size except OSError as e: - print(f"Error reading {file_path}: {str(e)}") + logging.error(f"Error reading {file_path}: {str(e)}") elif entry.is_dir(): # Recursively scan subdirectories scan_directory(entry.path) except OSError as e: - print(f"Error scanning directory {directory}: {str(e)}") + logging.error(f"Error scanning directory {directory}: {str(e)}") scan_directory(folder_path) return total_file_count, total_file_size, files_list @@ -544,9 +530,9 @@ def get_total_file_count_and_total_size(self, files_list, data_type): total_file_count += 1 total_file_size += file_size except OSError as e: - print(f"Error reading {file_path}: {str(e)}") + logging.error(f"Error reading {file_path}: {str(e)}") except Exception as e: - print(f"Unexpected error reading {file_path}: {str(e)}") + logging.error(f"Unexpected error reading {file_path}: {str(e)}") return total_file_count, total_file_size, files_list @@ -613,32 +599,7 @@ def validate_rotation_config(self, rotation_config): :param rotation_config: A dictionary containing the configuration for the rotations. :raises LabellerrError: If the configuration is invalid. """ - annotation_rotation_count = rotation_config.get("annotation_rotation_count") - review_rotation_count = rotation_config.get("review_rotation_count") - client_review_rotation_count = rotation_config.get( - "client_review_rotation_count" - ) - - # Validate review_rotation_count - if review_rotation_count != 1: - raise LabellerrError("review_rotation_count must be 1") - - # Validate client_review_rotation_count based on annotation_rotation_count - if annotation_rotation_count == 0 and client_review_rotation_count != 0: - raise LabellerrError( - "client_review_rotation_count must be 0 when annotation_rotation_count is 0" - ) - elif annotation_rotation_count == 1 and client_review_rotation_count not in [ - 0, - 1, - ]: - raise LabellerrError( - "client_review_rotation_count can only be 0 or 1 when annotation_rotation_count is 1" - ) - elif annotation_rotation_count > 1 and client_review_rotation_count != 0: - raise LabellerrError( - "client_review_rotation_count must be 0 when annotation_rotation_count is greater than 1" - ) + client_utils.validate_rotation_config(rotation_config) def _upload_preannotation_sync( self, project_id, client_id, annotation_format, annotation_file @@ -655,39 +616,20 @@ def _upload_preannotation_sync( """ try: # validate all the parameters - required_params = [ - "project_id", - "client_id", - "annotation_format", - "annotation_file", - ] - for param in required_params: - if param not in locals(): - raise LabellerrError(f"Required parameter {param} is missing") - - if annotation_format not in constants.ANNOTATION_FORMAT: - raise LabellerrError( - f"Invalid annotation_format. Must be one of {constants.ANNOTATION_FORMAT}" - ) - + required_params = { + "project_id": project_id, + "client_id": client_id, + "annotation_format": annotation_format, + "annotation_file": annotation_file, + } + client_utils.validate_required_params(required_params, list(required_params.keys())) + client_utils.validate_annotation_format(annotation_format, annotation_file) + url = f"{self.base_url}/actions/upload_answers?project_id={project_id}&answer_format={annotation_format}&client_id={client_id}" - - # validate if the file exist then extract file name from the path - if os.path.exists(annotation_file): - file_name = os.path.basename(annotation_file) - else: - raise LabellerrError("File not found") - - # Check if the file extension is .json when annotation_format is coco_json - if annotation_format == "coco_json": - file_extension = os.path.splitext(annotation_file)[1].lower() - if file_extension != ".json": - raise LabellerrError( - "For coco_json annotation format, the file must have a .json extension" - ) + file_name = client_utils.validate_file_exists(annotation_file) # get the direct upload url gcs_path = f"{project_id}/{annotation_format}-{file_name}" - print("Uploading your file to Labellerr. Please wait...") + logging.info("Uploading your file to Labellerr. Please wait...") direct_upload_url = self.get_direct_upload_url(gcs_path, client_id) # Now let's wait for the file to be uploaded to the gcs gcs.upload_to_gcs_direct(direct_upload_url, annotation_file) @@ -717,7 +659,7 @@ def _upload_preannotation_sync( self.job_id = job_id self.project_id = project_id - print(f"Preannotation upload successful. Job ID: {job_id}") + logging.info(f"Preannotation upload successful. Job ID: {job_id}") return self.preannotation_job_status() except Exception as e: logging.error(f"Failed to upload preannotation: {str(e)}") @@ -772,7 +714,7 @@ def upload_and_monitor(): ) # get the direct upload url gcs_path = f"{project_id}/{annotation_format}-{file_name}" - print("Uploading your file to Labellerr. Please wait...") + logging.info("Uploading your file to Labellerr. Please wait...") direct_upload_url = self.get_direct_upload_url(gcs_path, client_id) # Now let's wait for the file to be uploaded to the gcs gcs.upload_to_gcs_direct(direct_upload_url, annotation_file) @@ -802,7 +744,7 @@ def upload_and_monitor(): self.job_id = job_id self.project_id = project_id - print(f"Preannotation upload successful. Job ID: {job_id}") + logging.info(f"Preannotation upload successful. Job ID: {job_id}") # Now monitor the status headers = self._build_headers( @@ -817,13 +759,13 @@ def upload_and_monitor(): ) status_data = response.json() - print(" >>> ", status_data) + logging.debug(f"Status data: {status_data}") # Check if job is completed if status_data.get("response", {}).get("status") == "completed": return status_data - print("Syncing status after 5 seconds . . .") + logging.info("Syncing status after 5 seconds . . .") time.sleep(5) except Exception as e: @@ -867,7 +809,7 @@ def check_status(): if response_data.get("response", {}).get("status") == "completed": return response_data - print("retrying after 5 seconds . . .") + logging.info("retrying after 5 seconds . . .") time.sleep(5) except Exception as e: @@ -927,7 +869,7 @@ def upload_preannotation_by_project_id( "POST", url, headers=headers, data=payload, files=files ) response_data = self._handle_upload_response(response) - print("response_data -- ", response_data) + logging.debug(f"response_data: {response_data}") # read job_id from the response job_id = response_data["response"]["job_id"] @@ -935,7 +877,7 @@ def upload_preannotation_by_project_id( self.job_id = job_id self.project_id = project_id - print(f"Preannotation upload successful. Job ID: {job_id}") + logging.info(f"Preannotation upload successful. Job ID: {job_id}") future = self.preannotation_job_status_async() return future.result() @@ -944,13 +886,7 @@ def upload_preannotation_by_project_id( raise LabellerrError(f"Failed to upload preannotation: {str(e)}") def create_local_export(self, project_id, client_id, export_config): - unique_id = str(uuid.uuid4()) - required_params = [ - "export_name", - "export_description", - "export_format", - "statuses", - ] + unique_id = client_utils.generate_request_id() if project_id is None: raise LabellerrError("project_id cannot be null") @@ -961,22 +897,7 @@ def create_local_export(self, project_id, client_id, export_config): if export_config is None: raise LabellerrError("export_config cannot be null") - for param in required_params: - if param not in export_config: - raise LabellerrError(f"Required parameter {param} is missing") - if param == "export_format": - if export_config[param] not in constants.LOCAL_EXPORT_FORMAT: - raise LabellerrError( - f"Invalid export_format. Must be one of {constants.LOCAL_EXPORT_FORMAT}" - ) - if param == "statuses": - if not isinstance(export_config[param], list): - raise LabellerrError(f"Invalid statuses. Must be an array {param}") - for status in export_config[param]: - if status not in constants.LOCAL_EXPORT_STATUS: - raise LabellerrError( - f"Invalid status. Must be one of {constants.LOCAL_EXPORT_STATUS}" - ) + client_utils.validate_export_config(export_config) try: export_config.update( @@ -1002,7 +923,7 @@ def create_local_export(self, project_id, client_id, export_config): raise LabellerrError(f"Failed to create local export: {str(e)}") def fetch_download_url( - self, api_key, api_secret, project_id, uuid, export_id, client_id + self, project_id, uuid, export_id, client_id ): try: headers = self._build_headers( @@ -1034,9 +955,9 @@ def fetch_download_url( raise LabellerrError(f"Unexpected error in download_function: {str(e)}") def check_export_status( - self, api_key, api_secret, project_id, report_ids, client_id + self, project_id, report_ids, client_id ): - request_uuid = str(uuid.uuid4()) + request_uuid = client_utils.generate_request_id() try: if not project_id: raise LabellerrError("project_id cannot be null") @@ -1065,8 +986,6 @@ def check_export_status( # Download URL if job completed download_url = ( # noqa E999 todo check use of that self.fetch_download_url( - api_key=api_key, - api_secret=api_secret, project_id=project_id, uuid=request_uuid, export_id=status_item["report_id"], @@ -1193,9 +1112,9 @@ def initiate_create_project(self, payload): f"Invalid data_type. Must be one of {constants.DATA_TYPES}" ) - print("Rotation configuration validated . . .") + logging.info("Rotation configuration validated . . .") - print("Creating dataset . . .") + logging.info("Creating dataset . . .") dataset_response = self.create_dataset( { "client_id": payload["client_id"], @@ -1225,7 +1144,7 @@ def dataset_ready(): return True return False except Exception as e: - print(f"Error checking dataset status: {e}") + logging.error(f"Error checking dataset status: {e}") return False utils.poll( @@ -1235,7 +1154,7 @@ def dataset_ready(): timeout=60, ) - print("Dataset created and ready for use") + logging.info("Dataset created and ready for use") annotation_template_id = self.create_annotation_guideline( payload["client_id"], @@ -1243,7 +1162,7 @@ def dataset_ready(): payload["project_name"], payload["data_type"], ) - print("Annotation guidelines created") + logging.info("Annotation guidelines created") project_response = self.create_project( project_name=payload["project_name"], @@ -1324,8 +1243,8 @@ def upload_folder_files_to_dataset(self, data_config): f"Total file size: {total_file_volumn/1024/1024:.1f}MB exceeds limit of {constants.TOTAL_FILES_SIZE_LIMIT_PER_DATASET/1024/1024:.1f}MB" ) - print(f"Total file count: {total_file_count}") - print(f"Total file size: {total_file_volumn/1024/1024:.1f} MB") + logging.info(f"Total file count: {total_file_count}") + logging.info(f"Total file size: {total_file_volumn/1024/1024:.1f} MB") # Use generator for memory-efficient batch creation def create_batches(): @@ -1347,10 +1266,12 @@ def create_batches(): current_batch.append(file_path) current_batch_size += file_size except OSError as e: - print(f"Error accessing file {file_path}: {str(e)}") + logging.error(f"Error accessing file {file_path}: {str(e)}") fail_queue.append(file_path) except Exception as e: - print(f"Unexpected error processing {file_path}: {str(e)}") + logging.error( + f"Unexpected error processing {file_path}: {str(e)}" + ) fail_queue.append(file_path) if current_batch: @@ -1364,7 +1285,7 @@ def create_batches(): "No valid files found to upload in the specified folder" ) - print("CPU count", cpu_count(), " Batch Count", len(batches)) + logging.info(f"CPU count: {cpu_count()}, Batch Count: {len(batches)}") # Calculate optimal number of workers based on CPU count and batch count max_workers = min( @@ -1398,7 +1319,7 @@ def create_batches(): fail_queue.extend(batch) except Exception as e: logging.exception(e) - print(f"Batch upload failed: {str(e)}") + logging.error(f"Batch upload failed: {str(e)}") fail_queue.extend(batch) if not success_queue and fail_queue: diff --git a/labellerr/client_utils.py b/labellerr/client_utils.py new file mode 100644 index 0000000..ae416a1 --- /dev/null +++ b/labellerr/client_utils.py @@ -0,0 +1,169 @@ +""" +Shared utilities for both sync and async Labellerr clients. +""" +import uuid +from typing import Dict, Optional, Any +from . import constants + + +def build_headers( + api_key: str, + api_secret: str, + source: str = "sdk", + client_id: Optional[str] = None, + extra_headers: Optional[Dict[str, str]] = None, +) -> Dict[str, str]: + """ + Builds standard headers for API requests. + + :param api_key: API key for authentication + :param api_secret: API secret for authentication + :param source: Source identifier (e.g., "sdk", "sdk-async") + :param client_id: Optional client ID to include in headers + :param extra_headers: Optional dictionary of additional headers + :return: Dictionary of headers + """ + headers = { + "api_key": api_key, + "api_secret": api_secret, + "source": source, + "origin": constants.ALLOWED_ORIGINS, + } + + if client_id: + headers["client_id"] = str(client_id) + + if extra_headers: + headers.update(extra_headers) + + return headers + + +def validate_rotation_config(rotation_config: Dict[str, Any]) -> None: + """ + Validates a rotation configuration. + + :param rotation_config: A dictionary containing the configuration for the rotations. + :raises LabellerrError: If the configuration is invalid. + """ + from .exceptions import LabellerrError + + annotation_rotation_count = rotation_config.get("annotation_rotation_count") + review_rotation_count = rotation_config.get("review_rotation_count") + client_review_rotation_count = rotation_config.get("client_review_rotation_count") + + # Validate review_rotation_count + if review_rotation_count != 1: + raise LabellerrError("review_rotation_count must be 1") + + # Validate client_review_rotation_count based on annotation_rotation_count + if annotation_rotation_count == 0 and client_review_rotation_count != 0: + raise LabellerrError( + "client_review_rotation_count must be 0 when annotation_rotation_count is 0" + ) + elif annotation_rotation_count == 1 and client_review_rotation_count not in [0, 1]: + raise LabellerrError( + "client_review_rotation_count can only be 0 or 1 when annotation_rotation_count is 1" + ) + elif annotation_rotation_count > 1 and client_review_rotation_count != 0: + raise LabellerrError( + "client_review_rotation_count must be 0 when annotation_rotation_count is greater than 1" + ) + + +def validate_required_params(params: Dict[str, Any], required_list: list) -> None: + """ + Validates that all required parameters are present. + + :param params: Dictionary of parameters to validate + :param required_list: List of required parameter names + :raises LabellerrError: If any required parameter is missing + """ + from .exceptions import LabellerrError + + for param in required_list: + if param not in params: + raise LabellerrError(f"Required parameter {param} is missing") + + +def validate_file_exists(file_path: str) -> str: + """ + Validates that a file exists and returns the basename. + + :param file_path: Path to the file + :return: basename of the file + :raises LabellerrError: If file doesn't exist + """ + import os + from .exceptions import LabellerrError + + if os.path.exists(file_path): + return os.path.basename(file_path) + else: + raise LabellerrError(f"File not found: {file_path}") + + +def validate_annotation_format(annotation_format: str, annotation_file: str) -> None: + """ + Validates annotation format and file extension compatibility. + + :param annotation_format: Format of the annotation + :param annotation_file: Path to the annotation file + :raises LabellerrError: If format/extension mismatch + """ + import os + from .exceptions import LabellerrError + + if annotation_format not in constants.ANNOTATION_FORMAT: + raise LabellerrError( + f"Invalid annotation_format. Must be one of {constants.ANNOTATION_FORMAT}" + ) + + # Check if the file extension is .json when annotation_format is coco_json + if annotation_format == "coco_json": + file_extension = os.path.splitext(annotation_file)[1].lower() + if file_extension != ".json": + raise LabellerrError( + "For coco_json annotation format, the file must have a .json extension" + ) + + +def validate_export_config(export_config: Dict[str, Any]) -> None: + """ + Validates export configuration parameters. + + :param export_config: Export configuration dictionary + :raises LabellerrError: If configuration is invalid + """ + from .exceptions import LabellerrError + + required_params = [ + "export_name", + "export_description", + "export_format", + "statuses", + ] + + for param in required_params: + if param not in export_config: + raise LabellerrError(f"Required parameter {param} is missing") + + if param == "export_format": + if export_config[param] not in constants.LOCAL_EXPORT_FORMAT: + raise LabellerrError( + f"Invalid export_format. Must be one of {constants.LOCAL_EXPORT_FORMAT}" + ) + + if param == "statuses": + if not isinstance(export_config[param], list): + raise LabellerrError(f"Invalid statuses. Must be an array {param}") + for status in export_config[param]: + if status not in constants.LOCAL_EXPORT_STATUS: + raise LabellerrError( + f"Invalid status. Must be one of {constants.LOCAL_EXPORT_STATUS}" + ) + + +def generate_request_id() -> str: + """Generate a unique request ID.""" + return str(uuid.uuid4()) \ No newline at end of file From 072aa2ee3a36f642dbcf4c9693d882b0a3eb80f3 Mon Sep 17 00:00:00 2001 From: Gaurav Dudeja Date: Mon, 8 Sep 2025 13:40:32 +0530 Subject: [PATCH 3/6] test cases --- .env | 4 + .gitignore | 8 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 182 bytes labellerr/__pycache__/client.cpython-310.pyc | Bin 0 -> 31934 bytes .../__pycache__/exceptions.cpython-310.pyc | Bin 0 -> 393 bytes labellerr_use_case_tests.py | 535 ++++++++++++++++++ requirements.txt | 4 + 7 files changed, 551 insertions(+) create mode 100644 .env create mode 100644 labellerr/__pycache__/__init__.cpython-310.pyc create mode 100644 labellerr/__pycache__/client.cpython-310.pyc create mode 100644 labellerr/__pycache__/exceptions.cpython-310.pyc create mode 100644 labellerr_use_case_tests.py create mode 100644 requirements.txt diff --git a/.env b/.env new file mode 100644 index 0000000..d333843 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +export API_KEY=254034.58d3454c82aef27dd0cd928a93 +export API_SECRET=d81a759028ee62c59d5eaeb224d5a877a49f2de166431aa38cbaf0bebda99646 +export CLIENT_ID=12030 +export CLIENT_EMAIL=dev@labellerr.com \ No newline at end of file diff --git a/.gitignore b/.gitignore index a449859..f823d1d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,15 @@ build/ dist/ *.egg-info/ *.pyc +<<<<<<< Updated upstream .DS_Store .claude .idea tests/test_data +======= +*/*.pyc +.idea +.env + if file == '.DS_Store': + continue +>>>>>>> Stashed changes diff --git a/labellerr/__pycache__/__init__.cpython-310.pyc b/labellerr/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a735b8993434a23d5d37d6730bfcb26219de59b GIT binary patch literal 182 zcmd1j<>g`kf@xjrG7Nz9V-N=!FakLaKwQiLBvKfn7*ZI688n%ySPk?H^$h$p8E#mBE?C}IMt0~5bo^h1k*68hHyj>7YCPyszb=jSBI@ab;K%G zN3BwIY(8c6eL7v;7JQH6yNvS*tG~J(#{-{ETZ8jyYsea&*Q^m7jpEaPBU{~pt79m+ z6W`nLy{nn3?tX8|8n-51NLdrtwCWx!eI_-z{Z))WYUb56(#f=6exiQ9vAEcp)xYQB(3rRI6lRO*Yzo1e?$zPhh3IPDhOnQhm2Sk25jm#dlO z*8F6)YS@j-&Bm2#!LHREr#athSk=DS#b%@B)`CmL*~B%oK0Di3X4kTQwz*WFZ&b6F zn^vRkXX`6gvt2c<_Uy_Ms{6Xzc;21VJ$-6ndEUz+ZOr2Y_r;0ZoSVbR((>CSeF*85 z`ek0WnD3%yrkY!}+Y60Z*Edv;s>brl`Nih!WD)mnEw8wK)~@rWx$_vpM!n@5PGb=j z&5P8sV1_I zgGP#z7ZW#K&2aL*kgDp|+pVl+ypYD!eTQXQc^nxyDp*Av<*bs`hf=0hw)*j%w+5_1 zd>5>Da!R2@(HgNvkyEn9tZn%2Qxm9KwzgY4aNck2w07Zpz}ju?!S|q5vF^h6khRy^ zhwou)zx6hJk5~t+yYW40-D6GSd(67mx)0ymtb^7ee2=3~Q`R)@IAPsyJ%F6;)`QkV z_}*dVFiqd(_eV4HguHq0aG`>~;}tdRmASTEX|~*kU7vL^ew8at_d>7+ZUTB@TVOISt#Hp0BUi^~;AYH`~*X2IS>T&$g`wo0vZH==)Ev zx)<85>FCDO>dj3puli=K*2Iw3YSU=ZA$K-%M!Jyp#-q2VMs_MJxY56CHxAWXt+p$J z z^iI~wy`-%f9b>_;%!?WOh?{fG_597$we(ufEvy%8k#l#xdAZT5 z%rzGql~#SJvBezXwVGU0b78URxRv%?R~=3r{sq%V4STI=sTz+z%JRHh@IIpc9mWh> zv0JK`-Efw{0SgwU zULm8>$~aD^{)4>r1XxUw<&I2lK)zF5IbzTB`YSpVAg#zUvUukplqFc(~< z;;bw$w{16OEq3G5N(1BVg6Ui&(`i~*%M2`q?uujIgS-1Vyfp_akY5N2W88f+9L&jn zo0Hu)A31*JWDRdV@mwe*tvgsz8?Iketzsp$8W<|q_Vtw&w3TK0c0ae+p2ySA`}xN6 zT>QW{>@u3R$=+$3W-Zsr zfzI=w^9S9WRlu3K4U`Tl&XMLpy+!Xc5p|qU)7_~owr4SOII36-+A(QFCEqgLspTPh zPtF~X_FEy<{Yg>{sCHVOB0;A1E)1@{o5>y|ej)D8jK9roEP)j2_A2J1P(9`w=E5}1PMc|O zU+-MrqOGop4BVTFCh`JKnp`5ZM^n$0u>4@TTGu++o1C;RntOL`}@jz!DGLb9Ia z64c4ulu2u)C$Okti7`7SuSgq)YpmxwX2+OFvHnErrj|}!I?y4S%6NyO$$8pt&w@o^ z?$#^k>+bA@aP`1qL&nZO87+SdTx(^iejW=>K=X^u)Iy9O}<=sjq0GB3b5XPRAWxwKzV4lC8P{0I zt($nAY3F`i%MzcwtZMN}UalO)6>zZd%B8z79SY!f+D(nv!pdP1n3x3FPWEZ=Dm~>E zS1+bhYeqWd4ZEfFJ}d76k6a_4RA@y{=CF@eVa>Q|TuEO6r>n0)B#dUjIl(5*Ujmo2 z>Mqt;cMCWYT#2V)z(<*1b61xeiY@YpbC%hg%F4u%nY5XQ3b$7Cu>#?tj&*)%seZ_5 zko`h>tYG?aQ4i4+CtiXm-U>Q8Y}cD4v1qOdCOKSrTmi436LWP)%<&^^qz`&W;s>ZK ztpIAAZ&VtPY=BYf6@~R~-E*5ly*D(|j$fO)lQ;SpA*IzumjIwEjpspXhb!L5X*^=3 zsWR(Q>#!nzn`^r_@fdoDfx#*lc3D=v!8wP<`!D4^F(#(w@}gl!&T16jdolew68*eBWgdzlc# zDE5(V^e_m&q`I$I{52GG?nR;%3mUltc!XxA%}fcbogeLu5{`{@SsMY2F+Zfe?(OX9 z=~S}8E|x(aPhQVaa9T(rz@kJEkfW7xH8-=alc$3?b88tum@GaG_^mjcN0dPK!fH87(amxb2{)D^?|=!uk^990Ua z?n>Lf=q%S~8(UCnbS<8|30i$?)E%f(wzx}hC2@}==n0>v?o{4)=G07F$}3C<*PLmu zm_Rri#sOmIqcPH%)DwW?F1#z>I3+@by$>e}&=^Xb*wBDf`VvTmGv0oH$%9O~fr~xG zysdDHA#=A{iiQvm?iYd%<2VK`TSAtiKxV~4e+2h%Mv&~!>ww<^zF#XFh4eT9%8uy$ zD%zS#)SM-w2>-g)xUV4Bn{!hx;D|`QMAwv_`K{xda&wkbIFP30+yYctF!0F*@^pq$ zGtLWG9d);8tReu`FsS1_H+R`GJ$J}*b5i&^`e9Q9pH z+2`E;PIleAsY;b^$Z8o}&D!VPeyiUa=s-JqBO@yt)LG4RurBtmep|3E=G}qyL2IOA zglpo5Q`Ts-299~xA}TnyjCTUUp`2_7ItMHm&}u53OC_3}QI%+cI9{=uP)fC1z{}84 z;RGrREG%rxS%I$7aY7Niy$l6bx6;SkHyyO?Oow-Nrt6mVo|sgQZWIqhvhh*-4Bpzt zKz$kPSE-PusZ)(*YpZd&TehCe3i8@00kyi&JRRbqnwZ0TISuxxF{nHhKK==L{L}SS z>MageeEsyPGiP5>B7m=R4tocq!f1J}IS&;PpAO3hhZ9Szdi=SS#l_VrRHe-D>aQ^Q z1m@(cUq*u5)5!33O1>MTqT}lo)$O}c;+2eu-1a1nZSr$Jv*gTs2ZCWrzP2s8;@y2( zDRSd-s*@D|(enyd=LQ~gW^zcDw*;|oyW`r3?Ct*pS4z&mL0!8e20{r(5|0d z2Fh@ie3J`g{sgPwLMHy@E2S%GJ)=Qv2k+2~jZ(gx9@0#$fK_h^@yP5bKnXl&~hI4FQGkC zVdVfBn^#RDW9Sgr(lu`w_`4Le#CuZ+>$?FAzn1Uhtul}}TOO5`Kb&Z}|5h!#ed}ek zGXQ~&qcLa=-PBaO&?gS7J6R()4BVmM_Ad-rqYw>^PQe-j*DQ953xn1+jNa$vXdK+~ z=j3Pt-16sKty5g@1Gi+U?E$ys@dV0!9?!eO+6n%S80V5LmXT4Q{>cCwUxDI5nSt}l{eTqB| zL&Klop2&3TP!$K06i^!;8CkO8z_pB zK6V$@S4!N%CVQ&Mx@!2w`P;~`>>cgD~qm8c_SW;-X)=|h9vCke?wvCc_f-y z)*$(4+tURtSP#JMjSbV}nStZN>t$UeAt=GZJOe93DT_6O>%tqlW@rT|i<)VDLw{qa zP{2CUy^;)MOIlfr){@=57`3~DvsrX5{Nn;r#7E?+hp~u2<$~n{Di^Zvw5VJH)r>R5 z1trMINSPhT5nzZ~y5#Ifj*^^oHv=~MAkJY7TF2SVXesiZaiPdrHv|e(LgQkTBlIPN zMVjYw1WAk#x!3_Pez}OnCvBCYC8>|LRxC*{A1xFCQ4KKnK`5W0yIUw(L%8!tEq^sHDxVz*<@Q#R;(35q*8xA=%(ElxskAH!mML<4%qbIM;no_Q$n+IZL!#=P zNqgI)zC;R>dZ5YiCfKOxi$YyWbZJM7-Zs}8b7ld|H|LLb=LhV4LpxaUCSn#(HBExS z!|26s$kK4s8#(}>z0`zT0~F|skj)p5dZiGu0^=z#pI8d2nDaIjnttZJC(kOu^v(IC zw=-sh7)Q|p#H?^;a=?C^?F#G5u<^o~6Q@p}tUdAgnX_=tXg6Cua5u36^)NOPrX6#) zLadyDpFuGG_5~EGW@Y$#xtK8Ttq^QKCD!`lV#u%V1JOI*goX-vQ?xeJ-0))pv1-uR z+yfk&Xfc>4JOi*rnJh;m-VUxExvHSHB+hRsQEaXD;TMWcr;v)wh5TSn5foNNS(ohz{ z%`75;F%**^+6e7sVWmp&mtYgO(DX)pLRQIJaEV)Wmygm@PjsbSdJqk-RL6%?`Af8V zen+a4dd^%+Ur()Lskxc{KneyG&4RA|mGlhE&Ss-!IdnUDMf0_(`~3`Di;%7JPO)*q z&mWm@v>MMZ+s9g%z+d7mv=5D^yeE4XKTbY*f?6Ir z&?Mm5{2Y43+X?cqFQSexjLDKBahDm(Y-u3qcGO?LKanF%oMbuOZnt3`f+5+lA7gDQ zZUZxql#1EFckwdcf`9Lwj|+%F8KhyZ19BMwZ-6#e+Yap*P13XlmDBqLuV{Z(D0_#$ z{#Ve&8mK}eHWAj~;~-=W9WsRaM^o3KI9o~CeU}VxkDC(m2SLAJ!ax&~|1J=<;c5%e z41-mqq4mvJ*?{QzO`i@f*8Uq7?p(zpeyz`RlBzs*LQO0N}0*~+68RnbIEn4=M)EPtPW$6H@E*I(yo+e)4r$*01luQ}dlTo# zXW)_pMVnYkDj_3)x%F?d*e97h%;Y$e??K|{PMs015fix3duU}L;VQB1dwKM?nS6># zOsh9|Dzv&UL94fV5HCrjCJZlBG?7G;1R9cQA>%#tbqv-lxncNc;viB#{Si*Qvk3*a zID&zKuzVxci0YwU%FsGb34psy{&UQZ_*I#Z?D1n*+5gqwWBD zl}i|P;lml{i0W!^Eq6612!F`?(iXHD8WQJ19X50yj?~lQ~0ShpiP2SS1iN{j4m4Vtw6W^ z9hfh7s+{n4Z2`IiIRidDu$gY9NWL|)=5nGG3RFELFt+mSvMbDubpj&%?`PgEA@wfO zCn0qx{XiJ0MFP~{A!WW34QP4gEMauP>7lLQbgqnLHZYPN+KN(ddABaVsCvcplonYVDpJrXC=Ya+TQew~gAO6f-++??q-;gABCAsdx8V3?nUKFt@35O+ zp!L*%mkiv!K($cZLMErl=~+3V$>|9>>I2n2;g&ji`qxNKXmVnWK9qabEs4n~8=9O{ zSwQuIH3UWK@C2Z@xMk!Q`WDKoF9bk+*zI2*utv~N1=5vk2ITrUN2qv|kX@*Ftd5NW zN!JW4et~5w_J^&^!?f9egAS0cI5z}uc=Kv|CwkVPGRXg+NR1UPum@wNaIgZG<@O?7 ziUaj2asvJ7R((nmb>L$<1L-+pPH*Zj!AIUi=XX!X_k!!OXjW_VR9mEqHG&97?Po|7 zXOK*4)lyhs23|{kUR*9Ai#rMG7biH1-1b{mO~Q*ZxIbihZ#w1&fo&-q*tp~Iiz9~^ z?Bx)#Hd5ql(JYR#Yhz4$L~$|x9o#vo?*52{;g;9)CyhPU5GnMbq@NWHxjfE9G};Iv5S?ORpP7 zQhwL+L(oX}>JUxQIYvz9zFwwtn|pS_f-LUM0Xu<@a}`NwlUf6dSqBsVJAs`CN3`+i zl)&LBy3I{nSrRB%jsf4UeEoKH+O(dL>-h?u{`b1Bnrm}{_ z1n*BI-hOxkCDr<)YBx`hjndy4tVmm46+zeAv*oo&WA}h^g?iuZ8j1?vU%7(a7vNvv z-h!U~@UiZPl>rkrrS5*_w(ZWs=GAU3uF`kwetMcca0fSdz7~hAHPwXy2H7M&LXZyv!Fyl9X14UFDG6sGNB0wDyQym_kkuH-N242u zm3dk8`j>#lT@6?E;)?DP#VZ#P<-U>?)FAGZ!0o)616IJM51Por!}$`Y6Ki6x*c2GT6dwgszSTw7A> z#ZI9^*UchbGdo4J1^-+~86{wmflgst3KD+KEid$2gB|n`*ku487=>X9#xV`SFv^Ys zwe)rR)`!r8vS)-xFg%Yyz8_xwai9!rHT0lau*N!l3nSq;Slib6ul5s9Aa~Rn$4Gra zj%X16oE&Wj=J^~n64JWl>C9?t_JZARwPD|Fa{nSX0VA;LzZ#h@6bEwO;s{eq>4PJ(Vj~nI3IFR(7 zo#U2GoBBSbbj2=GdRv`*46Hud(3)6I2*~kL4~lnzybfXJ7wVTAm3l=g!y^}NShNXk zq^CF1hc@m%Cl1(D;;v%5lW4E0JdYS_w4>rmz3EO(O>Gno(7~SCvZD&NR@p&%6NHWg zF>Kt+OLrlunvEq0#K5W|<8D-vfK_&xQ%@d0Yts`eq0oAQ zr;qTKruZUbSm}j;SFE&K@8)%CYyFX60t6cq)ZV59o{k5LOOn!nmw6;kofRJaGbR_1 z_~TupRO8bId%_-K85xNCaRhiTt1l}l>HQTR{Wuc%i3JlK?+_E+fu9(^mAU^NEKwGF zKr~Z)GkoFx;HIrvu_u7v7t*oV(G*UXuJ^3>+rwQR-4Ek{g%+?FJ0AL=6{)Z*(1Hd} zfO2}LRzixr40Y{hdIVRo(}K#pJ}UOYO=0-n&~nCC=}|x(Ja7Vh1~y~+5_W;|zYJdj zY_)->K!o$}ji&wyWn;c|ce_vp-!?UXL_ZGa$BU^K(=Tc-W?t0ivKCOp3ntBX+<$X3 z_3=Jp3>fRM6X$8{61FlrY(m#EX-M)9q?|ohU{K@#v-nsvo$2*oOs+S_Ia!JSbuh$&eb$Yz=*b4Ta!oYZ*8|(>Rx%YCB21xM&69yX2=Gc+FYhUWu zwJObGT10>SI3mptLMRTL1F zJM_sN%Cdim-jV?jy{-d@X0H>78UjScDlI@1pc6neD?n5WfT(s|L%#>`5g^KA0irnp zq9y>SDNr;Y0ni@>bc6mb0H9_9fFl2DK>(-;I7!dNUxm91peTUlFUb)-7k>!=RN9oB z&~uUH2vmO+&q&Y3eB`;9kGvL(>}zeTQ$)J$dIl*0Xr;p~5)cIdK-tP&7^y!&D`Pl| zz_B}lCmVyRe{b}(`)=rKC7khX?Z@{xzTbxLZSvkw8m;Em`tm97kc`#coxV<~32$-$ zbZ*NNfR6j$(8MkFT7WM~-p4+=JJxqv2MKdA zq7_1Vz~K)DV>;~aTHkFQawj^_93>mRu*ngMF-(&LknfwHkf57A!CO^p}!bJ8zF`NvE|trp=E`=2mR z8~}foN4NKG`KK&GRH4>iIy4Jdh`rrmz3%pQ2k5M@*jXmg!oG3uEx=%~c6VVy^MHad zN6(5yz{^&S9XnQ0TfPy+BnXY-?We#&&=&fX&JnM^1KZ5O!YrpU#s62SdD^%qhFVa{ z_bd^(SqD78{M_*cxHK9oMk2{WHABXPLZ<2_cL9IVS&-$-iRKVj}PEmw6->M1dmv zaat_|JKnLgyc+I)=UVwnRuGacSgFE_f}aFQ#k$6!2+>=$r8a^~wCziTE$gVD6QcYD zV2ftn7PJ^i&|>Ne_8712g)&p4Z3IAO910Q>PL=~uXDVxn{-*;&;jwqN7u*C7(+$PW zuSRy8riIWrm}{{o$mYkNc}4dNn0K=)whgh@&&|p5>exDl1W^lC&M!6H(9F99B(;Bo z<;+G91xV?K{yp-Pt@oZg?c*F@10mQ*vD7rv``_^1{q;U>mld7_!DjjGJc>m8Dior_ z@*E*L*mSv%dP_vNV>6=jel}9WzBdTi5l}N!2E=Shr2z?5UPJ`?VyQf820m2)IF1Tn zQwjd!7c&*Smnj8*sJK-*w?II1O>T#YSwwW7S&TvSN5eq`V}2DUF1%0{XwQa2(}IQ% zVg_N6V=jGIGx+AlF8H1-P_LH7Hy4@QS{hm^4GY}??n$KgYX^R|V)f2+GNKFP4s}@8 zWO+kC6fAE=fR!U1c&wbkwq(K*Zv4WQ7Pi4; zPOvCtuVx7$E>$M`XY60WQ`!HT$-iOpZ<%m0QaoC%D8g;-;gx@f#M@1)@hUkq48xmQ zv(16Be+@~v>8HYsm|qI(6-}|}eu2p;CJ!+YF_4oL9`R&&icfCA z6X8D6!`w=<7aFq{gN5mfq^En)03sBKghf`P_WzODjAh`N4OSEHUe&(rJ=#lgUDG%r zvG%f+GIs(1g1kMKK8gbDjR+K$BEefze5FJuR$Txmks!PHypT$gx9rEw9T z6ZkmXL+1V1U$xR0VTYXrq=r0vNhD{6O7qMqq8*vWh|UY zpoZ=5pw$|7MIs$UxzSFU#3fj`v@m3i0Sm7K3vcU`7KYUosJv6_BWSn3)4x6%DP_l2 ze;K4YAf&o&ecajx{5X&xRkRW32&oRP4O|@{sZNg9Kb{nk&jpbMx+Y0rWd3WJI9ME_ z4NQMode}RK^uA`Fw_gHndE?3LrEU0_xrY!hp#crm7HA$kLO9&FqNaFy&Uk~N;4W&Y z%ArG*tx&xEv#2q-{Z0|DuftEjb!%iRytu+n2)h=>JJLCd( zkV61M@O(SY#efG>8_)ORTy4vt$?svR<2q`44xC}d;Abd33nMxKWSoHW4}#seI`j!A zP}g*Wo#wa$V>@ZK1s{7=#=vHwr%*=h?kW%W=wloxH;4MOxK?zr#YdSAFMZ(Be(#$B zap1CN^&!qCT!6#?l}8*F`UIG6ae*6s`T&>u#lQ$77;4k`i2NL;Jeu!r=pEAj{wrpdcOW$Kbai2q zfye#>7LK-Lh8>K{!606@?fR;>Cu~raiAw~VdhF-X5?tgL7vVFM7z}LW7yDtg6nYN9 zu;F;SrV-3;T6J!kdipMtl2VaW(V}=6K_3w#N-8BbN+*JS_-DBf{|@&;dV69Of^vEZ zLX)X(KyxtK>rDhklJbY2ICbLq6Y%sued?*Rp&{jQc8&zxbNLMHE$7bohVozP-QM_( ztRA~Yd$$qQt_VFRc54G~nhYAF zg2h1i!mjv*;_B1j7~CkRT#JDvJMdK^{}Vo!vl2N0$B@l~sNcW3(p9&-scs@CSr>>9 zqMOhHjgvwU5QyUWQ(^92CU-IU??}ACfVAMji}<-TOd@nkg_mVEJ%A%WZ#RT}Au3%) zL@VD=mbBPc?Ntp0u|>x~P!4EByzP}ogXUoD7J81NK6shRp^CR7rm*nFFxB!lh@N!! zCU<*mhzg6<=2lu_Y6ck&D1b7>{zm?M2%>f|K~AJhWm^ zL3NK7-3pr&7$$IJD*_stYajw#E$TE>Z{J4xIgF}MUWr#8l~~*jtmhFEBFNE&DvaXx z^tcw%K>|?h?FAH@Z!^n*2><$V5Rt==bpr^)`|eK0>BI6Tdwe^(J;I21ZWfFAIC2F{ zL}4OGEQo$bk_kM7bMVm@XT2ymgvsDE5X$r>bV3l-KnM`VOc)i4AS(FpLLe()k)DS) z4~})fuZwAW#wufxuC}1eZxygq7bF-B%k~FLHIK_!s^`%w#8n0{4+TrLlx4^UXh4SN zVFUo|*a6QUYD7k0^!YA18U^W&0kC|Rzz4Dez!v!YY(szu?DT!fSnIoV$pslhKm-&} zBo0z6EFd_<>d$~25pW!&Jh(n&?ZDz+PLOgYnG=v>`D$5q1r4wMMps<|hwi$GoMc^) z@~#>Ci+Fk7zKHnXm=Q!jsKZ;Lbx1{jpqD?wAgIadOshpVsbNA`#-<#aTHAKZim6RHts3-y!P!LIM#1$qHt<*6uN$?TTNXfD+>#*>g zFR_)aIEpZn{w{((wC)KOwRy=F4O>@#W=-@Z{2RbQ9-kBVI6sMm%QN^4d<3yzVn4qtTIP;EPs(MW zmK^k}200AqS?wQ!Xk%reb1jfxQ7HtTV}R$7Gr;=*fmd>!(&d!>0Z=`{JBHtD4474d z7y_1?;OPSo8GxDwA+tLDR!M^6^x-JaqnEi^xP0k2_8ozeAqth?t5EiiN2+2q zyRq>n^g)P&9IF`A3X=tqR$`qtXgI+7p~HP*7tN&MmqJN^F67n0x{E-{7qH)kb{s^1 zanyd%v4pEuQSstQW`EQZ|vLnZ=`PvgiN3jCzuH&u~6Y;YW! z|WaL?{KPqrX%xPj`5;`LJVKJAr z_y4l^GLs4uO6T_fW3s^HD@?=;CR7qH!qf?X)iB(2O>IVUs(E#a*i}kJ+yipMs2#z` z{yvF>-5JnT^rmpSn;UvTVOHa{0ql?;K^O^9wlff+9#c-t88ORCi1e_tnnh*%-2z-ha?LIes9lLiRrfXQW??WkZ zp}$nVo?b_YX3z)%u&*QddMCU5E-=-u z)(ouF2JZh1uBrKs9Ndk04dnH^67Y?baB0RGKsyMjU=6BoTpyBEjrnQ&s^sGv`NNVw zBKaS)M&*C(Mh15tIg+}L&G|e+q+M7yaD;W*rOW^Kqvp6PA(B`hyZa`@{*kw~^ZUBC zP(W#n2GSjpq9%@aLV3IE21XKNGI$dshVKDjg5BT{P*Kp+_M?IK(p*MJoZdcadbRW8r3DgppH>(c-TNHsyin8x%ih-$OT!MHc z(`ug+aLgb|KM|DfN9iH!ei1`bR-zW7AQwNZg|&b{9`L?1;%cXCFdJwkW#>dNvH6n> z%HtRDOyCf2VUN^&7z!i0crfX&8SPPV4n|FLcat)_h3$f!tt`i!5r)(X$fY;dk#GVs z3qK!inMx=(SjF2P>qxf3PAV*|@Dy+4TfgO7AAat>hb8^slzi>CudtPUL5OV#HZW|7 zUuOkjsUVXiSSeYQKagO*@~qy&L0vo>e>f!yIU8+>P`i{QY;K$vvToeQFSaG>lSFUu z?n|7|RX22Pl!y^geYF?J+mS3BK7Xi%SwnyA4j=E`CzN6-?Be-0!bymx1%VL)UqIP| z76ea-GQDp+sRHQs7KXcSZ`laD26wR(JEzzPq4I~=nRvg1aQ{o}+kvOT5Q$yOW+mMA zU@6BBZ%mfm+$2my zsDr1NwuY^g9bvt81?)KmhOnyxMvaXO(7jsNh+t(eioO6AC*uq}kN6WTAV_`n2*?p^ z#4Ed5KoHRkkSRz^5nQ>7Jx|!ZBu8TF$#wY}vO3EWx*gU0zThSZJzlp~ z-=|}YSV9x`Qd1jC8!(r`SW1Nu!y6v!6Usx)YB_4iBD=XXT@qdnfBAq_inr~k1W=g&Rg|k&#||By=Kb$hPT|Wuobaszj_4C z;I}HwP%R<9;bh^_@N+|^#Ms1S^>g&_X|K56U5_fj7V-|EEEz(2vsYH|yBTC^-mc>j zIma6rov;>oB1^pqPFiI{J5LAEI0I&t@CsjC1?^E{Q;|L?e|ZyK#fkss!f>75{!J1V zcpwN`oJys6l%E_{4pkye-Ot8}sbKDcgA<(fXD|2zr%sEfIU~hMTqOHPS`;uN(gpy2?*;=@(@Lv*%=rxN{? zL@}xqr5YqHl%X0mqm(~|ua&c*(>H^|sek=7(=gL;@Xilp^g)JzBP=yuGjju(0^nz| zTzDoWSpTb9iO1&G1`Gsp0v+;K7T{1$dc6A%=Y1D`9l;*^N~7Q|u^gaZV65M_w?fOn`O6>P_7Em@U^5=5m?I`B~ob}clCEictciz?t_!O-%Xtx;t%3164?v-(b#0s2W?vy&^ZBQRy z)SWN51H^*vAarKG-znO^1D)DXr%24#fw>J*ae1xOg0>5};y1*)kZa)>ap~u=Jkd3_ z6pY3;^l4j%hB!F=24fqLqoa-U{~A72(YL?nM6! zpGZ6J!T194Ljvf7_HE=^3I;gr^h2mGBJ@$_sB>%Awai+-TX4tLw=vQnLWYe)lHdJO z`s1~aYTjI)m%mZ>H9UJHyspz~y_z z&G;km&)U%$;>fn37Y&b}23=fg1!M5j!5D0JcdqXW0+97t`=ZfQZX=1h&q5C4d}ABo zcwQlRfW}nqnP}x7z#W`l>~dStk@IVvEXM2CFh2V^KG*q+C0XljU2?-0gZ^lpJT^&p zzpt+b*YL9&)`4UXOQ9Qq%UDvE0%Mp?5$L)fqgiAvJon&=pkh3tDN;1rLunPq{=mY>K z?{vHPD%gin#|M!L@sn3dJH-091-wR`gOxapAx;Ph)UTNZ;}ng$?SKz=p&|krEJO;g z5{u0Tan?GR2oRsKo8oV1m$Wm94aoq&A45U=t4IKuiirVN;7OLf!Vx|`ow1L7GDJ>@ zv-U{u3KDoXB!a9t6d{5a9|OC%Iq#k)8L<)bA(CW7izFfN5}kqi=|k+Niv$rERrFkm zd$;MQHzrPZ{q%;~z}2?#_@)5P=epF$6{vxLuF}byhPqlLsFpH_G;=L1(Edo*f_;U-`=ecPj621-N?{b>53r0%Q#@;aA_zO(ds8OW>Ej@Vu{f8@ORJeU_REhRbvFhGbhjbdy|FEJYFSg?fhn8&ImC~Qzi@+nyp|FG-i~b8aOCxK zHdNy+o{D=^%fTLI{4Nc2(oVH6;M+CWqE{N#0tZzc+uw`Es)LEBikI7qD@*tZU5+r7 z?T(1T_|;ibKm%ms77?n4 zcuIC_e;bnPoWQ?;106d%ha}|UKxPOs#gP1q69vpSME~VDgtShH<6bx@Z%^wPZV{y; za|u6N!Z3|2TY$WwLkbwDY!ZEIuNyk9tfwURV&SCo*w7;ku+Jdkg{Mt}+IxNLJy`6X7!DNC7;ztBqrT&N&UuEJk`6QDMGhvLe>O`Q^36_4f^O_+B9C@Uz z#>+R4sJD9z4Wyihkx*g|Wo8uug+Df@q8X9`WQI!_J!9Y;;nDGL{JCjjmS=~w{|`TY B^`8I$ literal 0 HcmV?d00001 diff --git a/labellerr/__pycache__/exceptions.cpython-310.pyc b/labellerr/__pycache__/exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d984744d59e2681d7f7080a6e313f00a46991cee GIT binary patch literal 393 zcmY*UOHRWu5VezthKeEVe3j1yX5qhX94lH9*}3$Wc|9iw^)EDw+LsIsgh&fOls0NJIl)7vk1xW5vC!j9i_p z7AQuTR=F-pD^gT?Cao59v{ilP23e1w?|lOrD`9*9 literal 0 HcmV?d00001 diff --git a/labellerr_use_case_tests.py b/labellerr_use_case_tests.py new file mode 100644 index 0000000..56c38f1 --- /dev/null +++ b/labellerr_use_case_tests.py @@ -0,0 +1,535 @@ + + +import os +import sys +import time +import json +import tempfile +import unittest +from unittest.mock import patch +from labellerr.client import LabellerrClient +from labellerr.exceptions import LabellerrError +import dotenv +dotenv.load_dotenv() + +class LabelerUseCaseIntegrationTests(unittest.TestCase): + + def setUp(self): + + self.api_key = os.getenv('API_KEY', 'test-api-key') + self.api_secret = os.getenv('API_SECRET', 'test-api-secret') + self.client_id = os.getenv('CLIENT_ID', 'test-client-id') + self.test_email = os.getenv('CLIENT_EMAIL', 'test@example.com') + + if (self.api_key == 'test-api-key' or + self.api_secret == 'test-api-secret' or + self.client_id == 'test-client-id' or + self.test_email == 'test@example.com'): + + raise ValueError( + "Real Labellerr credentials are required for integration testing. " + "Please set environment variables: " + "LABELLERR_API_KEY, LABELLERR_API_SECRET, LABELLERR_CLIENT_ID, LABELLERR_TEST_EMAIL" + ) + + # Initialize the client + self.client = LabellerrClient(self.api_key, self.api_secret) + + # Common test data + self.test_project_name = f"SDK_Test_Project_{int(time.time())}" + self.test_dataset_name = f"SDK_Test_Dataset_{int(time.time())}" + + # Sample annotation guide as per documentation requirements + self.annotation_guide = [ + { + "question": "What objects do you see?", + "option_type": "select", + "options": ["cat", "dog", "car", "person", "other"] + }, + { + "question": "Image quality rating", + "option_type": "radio", + "options": ["excellent", "good", "fair", "poor"] + } + ] + + # Valid rotation configuration + self.rotation_config = { + 'annotation_rotation_count': 1, + 'review_rotation_count': 1, + 'client_review_rotation_count': 1 + } + + def test_use_case_1_complete_project_creation_workflow(self): + + # Create temporary test files to simulate real data upload + test_files = [] + try: + # Create sample image files for testing + for i in range(3): + temp_file = tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) + temp_file.write(b'fake_image_data_' + str(i).encode()) + temp_file.close() + test_files.append(temp_file.name) + + # Step 1: Prepare project payload with all required parameters + project_payload = { + 'client_id': self.client_id, + 'dataset_name': self.test_dataset_name, + 'dataset_description': 'Test dataset for SDK integration testing', + 'data_type': 'image', + 'created_by': self.test_email, + 'project_name': self.test_project_name, + 'autolabel': False, + 'files_to_upload': test_files, + 'annotation_guide': self.annotation_guide, + 'rotation_config': self.rotation_config + } + + # Step 2: Execute complete project creation workflow + + result = self.client.initiate_create_project(project_payload) + + # Step 3: Validate the workflow execution + self.assertIsInstance(result, dict, "Project creation should return a dictionary") + self.assertEqual(result.get('status'), 'success', "Project creation should be successful") + self.assertIn('message', result, "Result should contain a success message") + self.assertIn('project_id', result, "Result should contain project_id") + + + # Store project details for potential cleanup + self.created_project_id = result.get('project_id') + self.created_dataset_name = self.test_dataset_name + + except LabellerrError as e: + self.fail(f"Project creation failed with LabellerrError: {e}") + except Exception as e: + self.fail(f"Project creation failed with unexpected error: {e}") + finally: + # Clean up temporary files + for file_path in test_files: + try: + os.unlink(file_path) + except OSError: + pass + + def test_use_case_1_validation_requirements(self): + """Table-driven test for project creation validation requirements""" + + validation_test_cases = [ + { + 'test_name': 'Missing client_id', + 'payload_overrides': {'client_id': None}, + 'remove_keys': ['client_id'], + 'expected_error': 'Required parameter client_id is missing' + }, + { + 'test_name': 'Invalid email format', + 'payload_overrides': {'created_by': 'invalid-email'}, + 'remove_keys': [], + 'expected_error': 'Please enter email id in created_by' + }, + { + 'test_name': 'Invalid data type', + 'payload_overrides': {'data_type': 'invalid_type'}, + 'remove_keys': [], + 'expected_error': 'Invalid data_type' + }, + { + 'test_name': 'Missing dataset_name', + 'payload_overrides': {}, + 'remove_keys': ['dataset_name'], + 'expected_error': 'Required parameter dataset_name is missing' + }, + { + 'test_name': 'Missing annotation guide and template ID', + 'payload_overrides': {}, + 'remove_keys': ['annotation_guide'], + 'expected_error': 'Please provide either annotation guide or annotation template id' + } + ] + + # Base valid payload + base_payload = { + 'client_id': self.client_id, + 'dataset_name': 'test_dataset', + 'dataset_description': 'test description', + 'data_type': 'image', + 'created_by': 'test@example.com', + 'project_name': 'test_project', + 'autolabel': False, + 'files_to_upload': [], + 'annotation_guide': self.annotation_guide + } + + for i, test_case in enumerate(validation_test_cases, 1): + with self.subTest(test_name=test_case['test_name']): + + # Create test payload by modifying base payload + test_payload = base_payload.copy() + test_payload.update(test_case['payload_overrides']) + + # Remove keys if specified + for key in test_case['remove_keys']: + test_payload.pop(key, None) + + # Execute test and verify expected error + with self.assertRaises(LabellerrError) as context: + self.client.initiate_create_project(test_payload) + + # Verify error message contains expected substring + error_message = str(context.exception) + self.assertIn(test_case['expected_error'], error_message, + f"Expected error '{test_case['expected_error']}' not found in '{error_message}'") + + def test_use_case_1_multiple_data_types_table_driven(self): + + project_test_scenarios = [ + { + 'scenario_name': 'Image Classification Project', + 'data_type': 'image', + 'file_extensions': ['.jpg', '.png'], + 'annotation_types': ['select', 'radio'], + 'expected_success': True + }, + { + 'scenario_name': 'Document Processing Project', + 'data_type': 'document', + 'file_extensions': ['.pdf'], + 'annotation_types': ['input', 'boolean'], + 'expected_success': True + } + ] + + test_scenario = project_test_scenarios[0] # Image classification + + + test_files = [] + try: + for ext in test_scenario['file_extensions'][:2]: # Limit to 2 files + temp_file = tempfile.NamedTemporaryFile(suffix=ext, delete=False) + temp_file.write(f'fake_{test_scenario["data_type"]}_data'.encode()) + temp_file.close() + test_files.append(temp_file.name) + + annotation_guide = [] + for i, annotation_type in enumerate(test_scenario['annotation_types']): + annotation_guide.append({ + "question": f"Test question {i+1}", + "option_type": annotation_type, + "options": ["option1", "option2", "option3"] if annotation_type in ['select', 'radio'] else [] + }) + + # Build project payload + project_payload = { + 'client_id': self.client_id, + 'dataset_name': f"SDK_Test_{test_scenario['data_type']}_{int(time.time())}", + 'dataset_description': f"Test dataset for {test_scenario['scenario_name']}", + 'data_type': test_scenario['data_type'], + 'created_by': self.test_email, + 'project_name': f"SDK_Test_Project_{test_scenario['data_type']}_{int(time.time())}", + 'autolabel': False, + 'files_to_upload': test_files, + 'annotation_guide': annotation_guide, + 'rotation_config': self.rotation_config + } + + # Execute test based on credentials + result = self.client.initiate_create_project(project_payload) + + self.assertIsInstance(result, dict) + self.assertEqual(result.get('status'), 'success') + print(f"✓ {test_scenario['scenario_name']} project created successfully") + + + finally: + # Clean up test files + for file_path in test_files: + try: + os.unlink(file_path) + except OSError: + pass + + def test_use_case_2_preannotation_upload_workflow(self): + annotation_data = { + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "bbox": [100, 100, 200, 200], + "area": 40000, + "iscrowd": 0 + } + ], + "images": [ + { + "id": 1, + "width": 640, + "height": 480, + "file_name": "test_image.jpg" + } + ], + "categories": [ + { + "id": 1, + "name": "person", + "supercategory": "human" + } + ] + } + + temp_annotation_file = None + try: + temp_annotation_file = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(annotation_data, temp_annotation_file) + temp_annotation_file.close() + + + test_project_id = 'test-project-id' + annotation_format = 'coco_json' + + + if hasattr(self, 'created_project_id') and self.created_project_id: + actual_project_id = self.created_project_id + else: + actual_project_id = test_project_id + + + print("Calling actual Labellerr pre-annotation API with real credentials...") + + try: + with patch.object(self.client, 'preannotation_job_status', create=True) as mock_status: + mock_status.return_value = { + 'response': { + 'status': 'completed', + 'job_id': 'real-job-id' + } + } + + result = self.client._upload_preannotation_sync( + project_id=actual_project_id, + client_id=self.client_id, + annotation_format=annotation_format, + annotation_file=temp_annotation_file.name + ) + + self.assertIsInstance(result, dict, "Upload should return a dictionary") + self.assertIn('response', result, "Result should contain response") + + + except Exception as api_error: + raise api_error + + + + except LabellerrError as e: + self.fail(f"Pre-annotation upload failed with LabellerrError: {e}") + except Exception as e: + self.fail(f"Pre-annotation upload failed with unexpected error: {e}") + finally: + if temp_annotation_file: + try: + os.unlink(temp_annotation_file.name) + except OSError: + pass + + def test_use_case_2_format_validation(self): + + format_test_cases = [ + { + 'test_name': 'Invalid annotation format', + 'project_id': 'test-project', + 'annotation_format': 'invalid_format', + 'annotation_file': 'test.json', + 'expected_error': 'Invalid annotation_format', + 'create_temp_file': False, + 'temp_suffix': None + }, + { + 'test_name': 'File not found', + 'project_id': 'test-project', + 'annotation_format': 'json', + 'annotation_file': 'non_existent_file.json', + 'expected_error': 'File not found', + 'create_temp_file': False, + 'temp_suffix': None + }, + { + 'test_name': 'Wrong file extension for COCO format', + 'project_id': 'test-project', + 'annotation_format': 'coco_json', + 'annotation_file': None, # Will be set to temp file + 'expected_error': 'For coco_json annotation format, the file must have a .json extension', + 'create_temp_file': True, + 'temp_suffix': '.txt' + } + ] + + for i, test_case in enumerate(format_test_cases, 1): + with self.subTest(test_name=test_case['test_name']): + + temp_file = None + try: + # Create temporary file if needed + if test_case['create_temp_file']: + temp_file = tempfile.NamedTemporaryFile( + suffix=test_case['temp_suffix'], + delete=False + ) + temp_file.write(b'test content') + temp_file.close() + annotation_file = temp_file.name + else: + annotation_file = test_case['annotation_file'] + + # Execute test and verify expected error + with self.assertRaises(LabellerrError) as context: + self.client._upload_preannotation_sync( + project_id=test_case['project_id'], + client_id=self.client_id, + annotation_format=test_case['annotation_format'], + annotation_file=annotation_file + ) + + # Verify error message contains expected substring + error_message = str(context.exception) + self.assertIn(test_case['expected_error'], error_message, + f"Expected error '{test_case['expected_error']}' not found in '{error_message}'") + + finally: + # Clean up temporary file + if temp_file: + try: + os.unlink(temp_file.name) + except OSError: + pass + + def test_use_case_2_multiple_formats_table_driven(self): + + preannotation_scenarios = [ + { + 'scenario_name': 'COCO JSON Upload', + 'annotation_format': 'coco_json', + 'file_extension': '.json', + 'sample_data': { + "annotations": [{"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 100, 100]}], + "images": [{"id": 1, "file_name": "test.jpg", "width": 640, "height": 480}], + "categories": [{"id": 1, "name": "test", "supercategory": "object"}] + }, + 'expected_success': True + }, + { + 'scenario_name': 'JSON Annotations Upload', + 'annotation_format': 'json', + 'file_extension': '.json', + 'sample_data': { + "labels": [{"image": "test.jpg", "annotations": [{"label": "cat", "confidence": 0.95}]}] + }, + 'expected_success': True + } + ] + + test_scenario = preannotation_scenarios[0] # COCO JSON + + + temp_annotation_file = None + try: + temp_annotation_file = tempfile.NamedTemporaryFile( + mode='w', + suffix=test_scenario['file_extension'], + delete=False + ) + json.dump(test_scenario['sample_data'], temp_annotation_file) + temp_annotation_file.close() + + + # Use project ID from previous tests if available + test_project_id = getattr(self, 'created_project_id', 'test-project-id-table-driven') + + + try: + # Only patch the missing method, let everything else be real + with patch.object(self.client, 'preannotation_job_status', create=True) as mock_status: + mock_status.return_value = { + 'response': { + 'status': 'completed', + 'job_id': f'job-{test_scenario["annotation_format"]}-{int(time.time())}' + } + } + + result = self.client._upload_preannotation_sync( + project_id=test_project_id, + client_id=self.client_id, + annotation_format=test_scenario['annotation_format'], + annotation_file=temp_annotation_file.name + ) + + self.assertIsInstance(result, dict) + + except Exception as api_error: + raise api_error + + + finally: + # Clean up annotation file + if temp_annotation_file: + try: + os.unlink(temp_annotation_file.name) + except OSError: + pass + + def tearDown(self): + pass + + @classmethod + def setUpClass(cls): + """Set up test suite.""" + + @classmethod + def tearDownClass(cls): + """Tear down test suite.""" + + +def run_use_case_tests(): + + # Create test suite + suite = unittest.TestLoader().loadTestsFromTestCase(LabelerUseCaseIntegrationTests) + + # Run tests with verbose output + runner = unittest.TextTestRunner(verbosity=2, stream=sys.stdout) + result = runner.run(suite) + + # Return success status + return result.wasSuccessful() + + +if __name__ == '__main__': + """ + Main execution block for running use case integration tests. + + Environment Variables Required: + - API_KEY: Your Labellerr API key + - API_SECRET: Your Labellerr API secret + - CLIENT_ID: Your Labellerr client ID + - TEST_EMAIL: Valid email address for testing + + Run with: + python use_case_tests.py + """ + # Check for required environment variables + required_env_vars = [ + 'API_KEY', + 'API_SECRET', + 'CLIENT_ID', + 'TEST_EMAIL' + ] + + missing_vars = [var for var in required_env_vars if not os.getenv(var)] + + + # Run the tests + success = run_use_case_tests() + + # Exit with appropriate code + sys.exit(0 if success else 1) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e286d93 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +python-dotenv +requests +pytest + From 39874db328a1d8af6537b0f4743c63ce5495ace9 Mon Sep 17 00:00:00 2001 From: Gaurav Dudeja Date: Mon, 8 Sep 2025 13:41:25 +0530 Subject: [PATCH 4/6] merge; --- .gitignore | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index f823d1d..2b8217e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,15 +4,11 @@ build/ dist/ *.egg-info/ *.pyc -<<<<<<< Updated upstream -.DS_Store -.claude -.idea -tests/test_data -======= */*.pyc .idea .env +.DS_Store +.claude +tests/test_data if file == '.DS_Store': continue ->>>>>>> Stashed changes From 575a27684ad03e4bc2758cceb8f4deea239da234 Mon Sep 17 00:00:00 2001 From: Gaurav Dudeja Date: Mon, 8 Sep 2025 13:45:29 +0530 Subject: [PATCH 5/6] remove --- .env | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index d333843..0000000 --- a/.env +++ /dev/null @@ -1,4 +0,0 @@ -export API_KEY=254034.58d3454c82aef27dd0cd928a93 -export API_SECRET=d81a759028ee62c59d5eaeb224d5a877a49f2de166431aa38cbaf0bebda99646 -export CLIENT_ID=12030 -export CLIENT_EMAIL=dev@labellerr.com \ No newline at end of file From 4f2acee9c0be2ca1a52413b54882d6f6af72ca46 Mon Sep 17 00:00:00 2001 From: Gaurav Dudeja Date: Fri, 12 Sep 2025 09:55:38 +0530 Subject: [PATCH 6/6] ci --- .github/workflows/ci.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a8896f..aad77dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,4 +39,33 @@ jobs: make format - name: Run tests run: | - make test \ No newline at end of file + make test + + integration-test: + name: Integration Tests + runs-on: ubuntu-20.04 + needs: test + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run integration tests + env: + LABELLERR_API_KEY: ${{ secrets.LABELLERR_API_KEY }} + LABELLERR_API_SECRET: ${{ secrets.LABELLERR_API_SECRET }} + LABELLERR_CLIENT_ID: ${{ secrets.LABELLERR_CLIENT_ID }} + LABELLERR_TEST_EMAIL: ${{ secrets.LABELLERR_TEST_EMAIL }} + run: | + python -m pytest labellerr_use_case_tests.py -v \ No newline at end of file