Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/workspace-import-migration.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a documented rate limit of 30 requests per second per workspace ([API rate limits](https://docs.databricks.com/aws/en/resources/limits)), higher than the limit that applied to the previous endpoint, and is ~1.5–2× faster for typical bundle deployments.
38 changes: 26 additions & 12 deletions acceptance/bin/fault.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
#!/usr/bin/env python3
"""Set up a fault rule on the testserver for the current test token.

Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]
Usage: fault.py [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]

PATTERN HTTP method and path, supports trailing * wildcard,
e.g. "PUT /api/2.0/permissions/pipelines/*"
STATUS_CODE HTTP status code to return, e.g. 504
OFFSET number of requests to let through before fault starts
TIMES number of times to return the fault response
ERROR_CODE optional error_code for the response body, e.g.
MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED)
PATTERN HTTP method and path, supports trailing * wildcard,
e.g. "PUT /api/2.0/permissions/pipelines/*"
STATUS_CODE HTTP status code to return, e.g. 504
OFFSET number of requests to let through before fault starts
TIMES number of times to return the fault response
ERROR_CODE optional error_code for the response body, e.g.
MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED)
--body-contains SUBSTR
only fire when the request body contains SUBSTR. Needed to
target a single file's /workspace/import upload, since every
upload shares the same method+path and only differs by the
multipart "path" form field.

The rule is scoped to the current DATABRICKS_TOKEN so it only affects
the test that registers it, even when tests share a server.
Expand All @@ -27,12 +32,20 @@
print("DATABRICKS_HOST not set", file=sys.stderr)
sys.exit(1)

if len(sys.argv) not in (5, 6):
print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr)
args = sys.argv[1:]
body_contains = ""
if len(args) >= 2 and args[0] == "--body-contains":
body_contains = args[1]
args = args[2:]

if len(args) not in (4, 5):
print(
f"usage: {sys.argv[0]} [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr
)
sys.exit(1)

pattern, status_code, offset, times = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
error_code = sys.argv[5] if len(sys.argv) == 6 else "INJECTED"
pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3])
error_code = args[4] if len(args) == 5 else "INJECTED"
body = json.dumps({"error_code": error_code, "message": "Fault injected by test."})

data = json.dumps(
Expand All @@ -42,6 +55,7 @@
"body": body,
"offset": offset,
"times": times,
"body_contains": body_contains,
}
).encode()

Expand Down
37 changes: 26 additions & 11 deletions acceptance/bundle/ai_runtime_task/local_code_source/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,29 @@ src/train.py

=== both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec

>>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create
>>> print_requests.py --sort //workspace/import //jobs/create
{
"method": "POST",
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
"q": {
"overwrite": "true"
"path": "/api/2.0/workspace/import",
"body": {
"multipart_form": {
"content": "[binary content 189 bytes]",
"format": "AUTO",
"overwrite": "true",
"path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
}
}
}
{
"method": "POST",
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
"q": {
"overwrite": "true"
"path": "/api/2.0/workspace/import",
"body": {
"multipart_form": {
"content": "[binary content 273 bytes]",
"format": "AUTO",
"overwrite": "true",
"path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
}
}
}
{
Expand Down Expand Up @@ -114,12 +124,17 @@ Deploying resources...
Updating deployment state...
Deployment complete!

>>> print_requests.py --sort --del-field raw_body //.air_snapshots/
>>> print_requests.py --sort //workspace/import
{
"method": "POST",
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
"q": {
"overwrite": "true"
"path": "/api/2.0/workspace/import",
"body": {
"multipart_form": {
"content": "[binary content 276 bytes]",
"format": "AUTO",
"overwrite": "true",
"path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
}
}
}

Expand Down
15 changes: 10 additions & 5 deletions acceptance/bundle/ai_runtime_task/local_code_source/script
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,23 @@ title "each task's tarball holds only synced files (both under the repo root .ai
trace list_code_snapshot.py

title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n"
# --del-field raw_body drops the binary tarball upload payload (kept readable). Filters
# use a leading // so Git Bash on Windows does not path-convert them. --keep is not
# passed, so print_requests.py consumes out.requests.txt.
trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create'
# Uploads go to POST /workspace/import, which carries the target filename in the
# multipart body rather than the URL, so .air_snapshots is matched with jq on the
# recorded body instead of by path filter. The tarball payload is binary and is
# already summarized as "[binary content N bytes]" by the request recorder.
# Filters use a leading // so Git Bash on Windows does not path-convert them.
# --keep is not passed, so print_requests.py consumes out.requests.txt.
trace print_requests.py --sort '//workspace/import' '//jobs/create' |
jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/|/jobs/create"))'

title "re-planning unchanged code is a no-op (no changes)\n"
trace $CLI bundle plan

title "editing a file changes the snapshot hash (content-addressed name changes)\n"
update_file.py src/train.py 'print("training")' 'print("training v2")'
trace $CLI bundle deploy
trace print_requests.py --sort --del-field raw_body '//.air_snapshots/'
trace print_requests.py --sort '//workspace/import' |
jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/"))'

title "destroy removes the deployed bundle (including the synced snapshots)\n"
trace $CLI bundle destroy --auto-approve
Expand Down
14 changes: 9 additions & 5 deletions acceptance/bundle/apps/app_yaml/out.app.yml.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
{
"method": "POST",
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml",
"q": {
"overwrite": "true"
},
"raw_body": "command:\n - python\n - app.py\n"
"path": "/api/2.0/workspace/import",
"body": {
"multipart_form": {
"content": "command:\n - python\n - app.py\n",
"format": "AUTO",
"overwrite": "true",
"path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml"
}
}
}
2 changes: 1 addition & 1 deletion acceptance/bundle/apps/app_yaml/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Deploying resources...
Updating deployment state...
Deployment complete!

>>> jq select(.path | test("app.yml")) out.requests.txt
>>> jq select(.body.multipart_form.path | strings | test("app.yml")) out.requests.txt

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/apps/app_yaml/script
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
trace $CLI bundle validate
trace $CLI bundle plan
trace $CLI bundle deploy
trace jq 'select(.path | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt
trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt
#trace print_requests.py //apps # currently fails due to TF inserting description=""
rm out.requests.txt

Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/artifacts/ai_runtime_code_source/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ Deployment complete!
}

=== Expecting the code_source tarball to be uploaded
>>> jq .path
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz"
>>> jq -r .body.multipart_form.path | strings
/Workspace/foo/bar/artifacts/.internal/code.tgz
2 changes: 1 addition & 1 deletion acceptance/bundle/artifacts/ai_runtime_code_source/script
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' out.requests.txt

title "Expecting the code_source tarball to be uploaded"
trace jq .path < out.requests.txt | grep import | grep code.tgz | sort
trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep code.tgz | sort

rm out.requests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ Deployment complete!
]

=== Expecting wheel to be uploaded
>>> jq .path
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
>>> jq -r .body.multipart_form.path | strings
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
/Workspace/foo/bar/artifacts/.internal/source.whl

=== Expecting environment dependencies to be updated
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create"
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt

title "Expecting wheel to be uploaded"
trace jq .path < out.requests.txt | grep import | grep whl | sort
trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort

title "Expecting environment dependencies to be updated"
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ Deploying resources...
Deployment complete!

=== Expecting wheel to be uploaded
>>> jq .path
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
>>> jq -r .body.multipart_form.path | strings
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
/Workspace/foo/bar/artifacts/.internal/source.whl

=== Expecting delete request to artifact_path/.internal folder
>>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ echo "test wheel content" > whl/source.whl
trace $CLI bundle deploy

title "Expecting wheel to be uploaded"
trace jq .path < out.requests.txt | grep import | grep whl | sort
trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort

title "Expecting delete request to artifact_path/.internal folder"
trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt
Expand Down
18 changes: 9 additions & 9 deletions acceptance/bundle/artifacts/upload_multiple_libraries/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ Deployment complete!
]

=== Expecting 4 wheels to be uploaded
>>> jq .path
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl"
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl"
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl"
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl"
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl"
>>> jq -r .body.multipart_form.path | strings
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl
/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl
/Workspace/foo/bar/artifacts/.internal/source1.whl
/Workspace/foo/bar/artifacts/.internal/source2.whl
/Workspace/foo/bar/artifacts/.internal/source3.whl
/Workspace/foo/bar/artifacts/.internal/source4.whl

=== Expecting environment dependencies to be updated
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create"
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt

title "Expecting 4 wheels to be uploaded"
trace jq .path < out.requests.txt | grep import | grep whl | sort
trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort

title "Expecting environment dependencies to be updated"
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt
Expand Down
Loading
Loading