From 15abe6da27dda3954a3f2b59b3686facb2b306f7 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 3 Aug 2026 14:02:18 +0000 Subject: [PATCH 1/5] initial filesystem changes --- acceptance/internal/prepare_server.go | 58 +++++ libs/filer/workspace_files_client.go | 117 +++++++--- libs/filer/workspace_files_client_test.go | 249 ++++++++++++++++++++++ libs/testserver/fake_workspace.go | 37 +++- libs/testserver/handlers.go | 75 +++++++ 5 files changed, 497 insertions(+), 39 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 3b1be7a79b3..de943a02197 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -1,8 +1,12 @@ package internal import ( + "bytes" "encoding/json" "fmt" + "io" + "mime" + "mime/multipart" "net/http" "os" "path/filepath" @@ -307,6 +311,11 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR if json.Valid(req.Body) { result.Body = json.RawMessage(req.Body) + } else if normalized, ok := normalizeMultipartBody(req); ok { + // Multipart bodies contain a randomly generated boundary string and binary + // content; record a normalized form (sorted form-field names with sizes for + // file parts) so recorded requests stay deterministic and reviewable. + result.Body = normalized } else { result.RawBody = string(req.Body) } @@ -314,6 +323,55 @@ func getLoggedRequest(req *testserver.Request, includedHeaders []string) LoggedR return result } +// normalizeMultipartBody returns a deterministic representation of a multipart +// form body if the request's Content-Type is multipart/*. The second return +// value is false if the body is not multipart or cannot be parsed. +// +// Text parts are recorded as their literal content so reviewers can read what +// was uploaded; existing global UNIX_TIME / UUID / USERNAME replacements +// normalize the timestamp / id / email fields embedded in deploy.lock and +// similar payloads. Non-UTF-8 (binary) parts and parts whose serialized +// content would bloat the recording past multipartContentLimit are +// summarized as "[binary content N bytes]". +func normalizeMultipartBody(req *testserver.Request) (any, bool) { + contentType := req.Headers.Get("Content-Type") + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil || !strings.HasPrefix(mediaType, "multipart/") { + return nil, false + } + boundary := params["boundary"] + if boundary == "" { + return nil, false + } + mr := multipart.NewReader(bytes.NewReader(req.Body), boundary) + parts := map[string]any{} + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false + } + name := part.FormName() + if !utf8.Valid(data) || len(data) > multipartContentLimit { + parts[name] = fmt.Sprintf("[binary content %d bytes]", len(data)) + continue + } + parts[name] = string(data) + } + return map[string]any{"multipart_form": parts}, true +} + +// Multipart parts larger than this limit are summarized as a size placeholder +// to keep recorded fixtures small. Reviewers care about *what* was uploaded +// (path, format, overwrite flag), not the bytes themselves, for large blobs. +const multipartContentLimit = 4096 + func filterHeaders(h http.Header, includedHeaders []string) http.Header { headers := make(http.Header) for k, v := range h { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index e1b62481243..5b903ab3678 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,7 +9,6 @@ import ( "io" "io/fs" "net/http" - "net/url" "path" "slices" "strings" @@ -23,6 +22,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +// workspaceObjectTypeMismatchReason is the AIP-193 ErrorInfo reason attached +// by /workspace/import when overwrite=true targets a path whose existing +// object's node type differs from the upload (FILE vs NOTEBOOK). +const workspaceObjectTypeMismatchReason = "WORKSPACE_OBJECT_TYPE_MISMATCH" + // Type that implements fs.DirEntry for WSFS. type wsfsDirEntry struct { wsfsFileInfo @@ -154,35 +158,42 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Remove leading "/" so we can use it in the URL. - overwrite := slices.Contains(mode, OverwriteIfExists) - urlPath := fmt.Sprintf( - "/api/2.0/workspace-files/import-file/%s?overwrite=%t", - url.PathEscape(strings.TrimLeft(absPath, "/")), - overwrite, - ) - // Buffer the file contents because we may need to retry below and we cannot read twice. body, err := io.ReadAll(reader) if err != nil { return err } - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, w.workspaceIDHeaders(), nil, body, nil) + // Use Workspace.Upload (multipart /api/2.0/workspace/import) instead of the + // JSON-body variant of the same endpoint, which caps payloads at 10 MiB for + // AUTO format (databricks.webapp.autoExportFormatLimitBytes). The multipart + // variant has been verified against a real workspace at 450 MB for regular + // files — strictly better than the previous /workspace-files/import-file + // endpoint, which has a 200 MiB body cap + // (databricks.workspaceFilesystem.maxImportSizeBytes) plus a 305s server-side + // request timeout that cuts off uploads above ~400 MB at typical bandwidth. + // + // Notebook content (any payload with a `# Databricks notebook source` header + // detected by format=AUTO) hits a separate 10 MiB cap on the server + // (databricks.notebook.maxNotebookSizeBytes). Both /workspace-files/import-file + // and /workspace/import enforce this same cap, so switching from the former + // to the latter does not regress maximum notebook upload size. + overwrite := slices.Contains(mode, OverwriteIfExists) + uploadOpts := []func(*workspace.Import){ + workspace.UploadFormat(workspace.ImportFormatAuto), + } + if overwrite { + uploadOpts = append(uploadOpts, workspace.UploadOverwrite()) + } + err = w.workspaceClient.Workspace.Upload(ctx, absPath, bytes.NewReader(body), uploadOpts...) // Return early on success. if err == nil { return nil } - // Special handling of this error only if it is an API error. - aerr, ok := errors.AsType[*apierr.APIError](err) - if !ok { - return err - } - - // This API returns a 404 if the parent directory does not exist. - if aerr.StatusCode == http.StatusNotFound { + // Parent directory does not exist. + if errors.Is(err, apierr.ErrNotFound) { if !slices.Contains(mode, CreateParentDirectories) { return noSuchDirectoryError{path.Dir(absPath)} } @@ -190,8 +201,8 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { - if mkdirErr, ok := errors.AsType[*apierr.APIError](err); ok && mkdirErr.StatusCode == http.StatusForbidden { - return permissionError{absPath, mkdirErr} + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -200,24 +211,62 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return w.Write(ctx, name, bytes.NewReader(body), sliceWithout(mode, CreateParentDirectories)...) } - // This API returns 409 if the file already exists, when the object type is file - if aerr.StatusCode == http.StatusConflict { + // Path already taken. /workspace/import returns this in three shapes, + // verified against a real workspace: + // + // - 400 RESOURCE_ALREADY_EXISTS — sequential conflict, no overwrite flag. + // Example: "/Users/me/foo.txt already exists. Please pass overwrite=true + // to overwrite it." + // + // - 409 ALREADY_EXISTS — concurrent contention (observed in TestLock when + // five lockers race to write deploy.lock). + // Example: "Node with name /Users/me/.bundle/.../deploy.lock already + // exists. Please pass overwrite=true to update it." + // + // - 400 INVALID_PARAMETER_VALUE — overwrite=true on a path where the + // existing object's node type differs from the upload. Two distinct + // messages, both observed against aws-prod-ucws: + // + // (a) "Cannot overwrite the asset at /Users/me/foo due to type mismatch + // (asked: FILE, actual: NOTEBOOK)" — fires when the upload path is + // the same as an existing NOTEBOOK and the new content has no + // notebook header (so AUTO would store it as FILE), or the mirror + // case with FILE/NOTEBOOK swapped. + // + // (b) "Requested node type [FILE] is different from the existing node + // type [NOTEBOOK]" — fires when /foo is already a NOTEBOOK (from a + // prior /foo.py upload) and an overwrite-upload of regular content + // targets /foo.py: AUTO would store the new content as FILE at + // /foo.py, but the workspace treats /foo.py as the source view of + // the existing /foo NOTEBOOK and rejects the type change. + // + // The server refuses the overwrite even though the caller asked for + // it; from the caller's perspective the path is occupied, so we + // surface this as already-exists. + if errors.Is(err, apierr.ErrResourceAlreadyExists) || errors.Is(err, apierr.ErrAlreadyExists) { return fileAlreadyExistsError{absPath} } - - // This API returns 400 if the file already exists when the object type is notebook. - // Both the historical "Path () already exists." format and the newer - // "RESOURCE_ALREADY_EXISTS: already exists. ..." format end with the same - // "already exists." marker; the JSON error_code is empty in both. The new format - // might not have been rolled out to all workspaces yet, so we anchor on the shared - // marker and return absPath rather than parsing the message. - if aerr.StatusCode == http.StatusBadRequest && strings.Contains(aerr.Message, "already exists.") { - return fileAlreadyExistsError{absPath} + if errors.Is(err, apierr.ErrInvalidParameterValue) { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + // WCS attaches AIP-193 ErrorInfo with a stable reason to import + // path collisions (universe PR #2019174, WP-6031), so prefer + // branching on it over parsing the message. + if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == workspaceObjectTypeMismatchReason { + return fileAlreadyExistsError{absPath} + } + // Fallback for workspaces where the ErrorInfo change has not + // rolled out: as of 2026-06-12 aws-prod-ucws still returns these + // errors without details, so match the two observed messages. + // Remove once the rollout is confirmed everywhere. + if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { + return fileAlreadyExistsError{absPath} + } + } } - // This API returns StatusForbidden when you have read access but don't have write access to a file - if aerr.StatusCode == http.StatusForbidden { - return permissionError{absPath, aerr} + // Caller has read access but no write access. + if errors.Is(err, apierr.ErrPermissionDenied) { + return permissionError{absPath, err} } return err diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index 2915af60436..a607b116263 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -1,8 +1,13 @@ package filer import ( + "bytes" + "context" "encoding/json" + "io" "io/fs" + "net/http" + "strings" "testing" "time" @@ -10,8 +15,10 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -97,6 +104,248 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { + tests := []struct { + name string + modes []WriteMode + expectOverride bool + }{ + { + name: "no overwrite", + modes: nil, + expectOverride: false, + }, + { + name: "overwrite", + modes: []WriteMode{OverwriteIfExists}, + expectOverride: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + + workspaceApi.EXPECT().Upload( + mock.Anything, + "/dir/file.txt", + mock.Anything, + mock.Anything, + mock.Anything, + ).RunAndReturn(func(_ context.Context, _ string, r io.Reader, opts ...func(*workspace.Import)) error { + body, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, "hello", string(body)) + + i := &workspace.Import{} + for _, opt := range opts { + opt(i) + } + assert.Equal(t, workspace.ImportFormatAuto, i.Format) + assert.Equal(t, tc.expectOverride, i.Overwrite) + return nil + }).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "file.txt", strings.NewReader("hello"), tc.modes...) + require.NoError(t, err) + }) + } +} + +func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { + tests := []struct { + name string + mode []WriteMode + apiErr *apierr.APIError + expectErrTarget any + }{ + { + name: "404 without create-parent maps to noSuchDirectoryError", + apiErr: &apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}, + expectErrTarget: noSuchDirectoryError{}, + }, + { + name: "400 RESOURCE_ALREADY_EXISTS maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "RESOURCE_ALREADY_EXISTS", + Message: "/dir/file.txt already exists. Please pass overwrite=true to overwrite it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "409 ALREADY_EXISTS (concurrent contention) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusConflict, + ErrorCode: "ALREADY_EXISTS", + Message: "Node with name /dir/file.txt already exists. Please pass overwrite=true to update it.", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE 'type mismatch' (overwrite=true) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Cannot overwrite the asset at /dir/foo due to type mismatch (asked: FILE, actual: NOTEBOOK).", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE 'Requested node type' (overwrite=true) maps to fileAlreadyExistsError", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "Requested node type [FILE] is different from the existing node type [NOTEBOOK]", + }, + expectErrTarget: fileAlreadyExistsError{}, + }, + { + name: "400 INVALID_PARAMETER_VALUE other message passes through", + apiErr: &apierr.APIError{ + StatusCode: http.StatusBadRequest, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + expectErrTarget: nil, + }, + { + name: "403 maps to permissionError", + apiErr: &apierr.APIError{StatusCode: http.StatusForbidden, Message: "denied"}, + expectErrTarget: permissionError{}, + }, + { + name: "500 passes through", + apiErr: &apierr.APIError{StatusCode: http.StatusInternalServerError, Message: "boom"}, + expectErrTarget: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(tc.apiErr).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "file.txt", bytes.NewReader([]byte("data")), tc.mode...) + require.Error(t, err) + switch target := tc.expectErrTarget.(type) { + case noSuchDirectoryError: + assert.ErrorAs(t, err, &target) + case fileAlreadyExistsError: + assert.ErrorAs(t, err, &target) + case permissionError: + assert.ErrorAs(t, err, &target) + case nil: + // passthrough — same APIError pointer + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, tc.apiErr.StatusCode, aerr.StatusCode) + } + }) + } +} + +// writeWithImportError exercises Write through a real HTTP roundtrip so the +// SDK parses AIP-193 error details from the response body (the errorDetails +// field on apierr.APIError is unexported and only populated during response +// parsing, so it cannot be set on a directly constructed APIError). +func writeWithImportError(t *testing.T, body map[string]any) error { + t.Helper() + + server := testserver.New(t) + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: http.StatusBadRequest, + Body: body, + } + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewWorkspaceFilesClient(client, "/dir") + require.NoError(t, err) + + err = f.Write(t.Context(), "file.txt", strings.NewReader("data"), OverwriteIfExists) + require.Error(t, err) + return err +} + +func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { + // The message is deliberately one the fallback string match does not + // recognize, to prove the branch fires on the structured reason alone. + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some future wording for the same condition", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": workspaceObjectTypeMismatchReason, + "domain": "workspace.databricks.com", + "metadata": map[string]string{"existing_type": "NOTEBOOK"}, + }, + }, + }) + var target fileAlreadyExistsError + assert.ErrorAs(t, err, &target) +} + +func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { + err := writeWithImportError(t, map[string]any{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "some other validation failure", + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "SOME_OTHER_REASON", + "domain": "workspace.databricks.com", + }, + }, + }) + var aerr *apierr.APIError + require.ErrorAs(t, err, &aerr) + assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) +} + +func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { + mw := mocks.NewMockWorkspaceClient(t) + workspaceApi := mw.GetMockWorkspaceAPI() + + // First Upload returns 404, second returns success after MkdirsByPath. + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(&apierr.APIError{StatusCode: http.StatusNotFound, Message: "not found"}).Once() + + workspaceApi.EXPECT().MkdirsByPath(mock.Anything, "/dir/sub").Return(nil).Once() + + workspaceApi.EXPECT().Upload( + mock.Anything, "/dir/sub/file.txt", mock.Anything, mock.Anything, mock.Anything, + ).Return(nil).Once() + + c := WorkspaceFilesClient{ + workspaceClient: mw.WorkspaceClient, + root: NewWorkspaceRootPath("/dir"), + } + err := c.Write(t.Context(), "sub/file.txt", strings.NewReader("data"), CreateParentDirectories) + require.NoError(t, err) +} + func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 404742132af..ff29821cf14 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -510,6 +510,32 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) { } } +// detectNotebookLanguage mirrors the real /workspace/import format=AUTO logic: +// a file whose extension is .py / .sql / .scala / .r and whose content starts +// with the language-appropriate "Databricks notebook source" line-comment is +// stored as a NOTEBOOK with the corresponding language; otherwise it's a FILE. +func detectNotebookLanguage(extension string, body []byte) (workspace.Language, bool) { + switch extension { + case ".py": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguagePython, true + } + case ".sql": + if bytes.HasPrefix(body, []byte("-- Databricks notebook source")) { + return workspace.LanguageSql, true + } + case ".scala": + if bytes.HasPrefix(body, []byte("// Databricks notebook source")) { + return workspace.LanguageScala, true + } + case ".r": + if bytes.HasPrefix(body, []byte("# Databricks notebook source")) { + return workspace.LanguageR, true + } + } + return "", false +} + func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, overwrite bool) Response { if !strings.HasPrefix(filePath, "/") { filePath = "/" + filePath @@ -532,18 +558,19 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o } } - // Note: Files with .py, .scala, .r or .sql extension can - // be notebooks if they contain a magical "Databricks notebook source" - // header comment. We omit support non-python extensions for now for simplicity. + // Files with .py / .sql / .scala / .r extension can be notebooks if they + // carry the "Databricks notebook source" header comment in the language's + // line-comment syntax. Mirror the real workspace's auto-detection here so + // acceptance tests can assert the resulting object_type via /workspace/list. extension := filepath.Ext(filePath) - if extension == ".py" && strings.HasPrefix(string(body), "# Databricks notebook source") { + if lang, isNotebook := detectNotebookLanguage(extension, body); isNotebook { // Notebooks are stripped of their extension by the workspace import API. workspacePath = strings.TrimSuffix(filePath, extension) s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ ObjectType: "NOTEBOOK", Path: workspacePath, - Language: "PYTHON", + Language: lang, ObjectId: nextID(), }, Data: body, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index f98dd944d1f..5a0a95acfd6 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -153,6 +153,81 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("POST", "/api/2.0/workspace/import", func(req Request) any { + // /workspace/import accepts both a JSON body (matching workspace.Import) and a + // multipart form body. The multipart variant is what databricks-sdk-go's + // Workspace.Upload uses; the JSON variant is kept for back-compat with anything + // that still hits Workspace.Import directly. + contentType := req.Headers.Get("Content-Type") + mediaType, params, _ := mime.ParseMediaType(contentType) + if strings.HasPrefix(mediaType, "multipart/") { + mr := multipart.NewReader(bytes.NewReader(req.Body), params["boundary"]) + var ( + filePath string + content []byte + format string + overwrite bool + ) + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + data, err := io.ReadAll(part) + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + switch part.FormName() { + case "path": + filePath = string(data) + case "content": + content = data + case "format": + format = string(data) + case "overwrite": + overwrite = string(data) == "true" + } + } + + if format != "" && format != string(workspace.ImportFormatAuto) { + return Response{ + Body: "internal error: The test server only supports auto format.", + StatusCode: http.StatusInternalServerError, + } + } + + // Translate any 409 from the shared fake into the 400 + errorCode + // RESOURCE_ALREADY_EXISTS shape returned by the real /workspace/import + // endpoint, including the AIP-193 ErrorInfo detail it attaches to path + // collisions (universe PR #2019174, WP-6031). + resp := req.Workspace.WorkspaceFilesImportFile(filePath, content, overwrite) + if resp.StatusCode == http.StatusConflict { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Path (%s) already exists.", filePath), + "details": []map[string]any{ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "RESOURCE_ALREADY_EXISTS", + "domain": "workspace.databricks.com", + }, + }, + }, + } + } + return resp + } + var request workspace.Import err := json.Unmarshal(req.Body, &request) if err != nil { From d730ec084a5998d17712e453116e2bc07633e1eb Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 3 Aug 2026 15:25:34 +0000 Subject: [PATCH 2/5] filer: switch workspace upload from import-file to /workspace/import Replace POST /api/2.0/workspace-files/import-file/{path} with the multipart variant of POST /api/2.0/workspace/import (via the SDK's Workspace.Upload + format=AUTO). The previous endpoint is deprecated; the new one has a higher rate limit (30 vs 20 rps/workspace) and is ~1.5-2x faster for typical bundle deployments. Error handling is rewritten to branch on SDK sentinels (errors.Is against ErrNotFound / ErrResourceAlreadyExists / ErrAlreadyExists / ErrInvalidParameterValue / ErrPermissionDenied) and the AIP-193 ErrorInfo reason WORKSPACE_OBJECT_TYPE_MISMATCH, with a message-substring fallback for workspaces where WP-6031 has not rolled out. The testserver now decodes the multipart /workspace/import body and mirrors the real endpoint's format=AUTO notebook detection. A new bodyContains option on the fault mechanism lets a test target a single file's upload, which is no longer possible by URL since every upload shares the /workspace/import path. Co-authored-by: Isaac --- .../bundles/workspace-import-migration.md | 1 + acceptance/bin/fault.py | 38 +++-- .../bundle/apps/app_yaml/out.app.yml.txt | 14 +- acceptance/bundle/apps/app_yaml/output.txt | 2 +- acceptance/bundle/apps/app_yaml/script | 2 +- .../ai_runtime_code_source/output.txt | 4 +- .../artifacts/ai_runtime_code_source/script | 2 +- .../artifact_upload_for_workspace/output.txt | 6 +- .../artifact_upload_for_workspace/script | 2 +- .../output.txt | 6 +- .../script | 2 +- .../upload_multiple_libraries/output.txt | 18 +-- .../upload_multiple_libraries/script | 2 +- .../artifacts/whl_change_version/output.txt | 78 ++++----- .../artifacts/whl_change_version/script | 6 +- .../bundle/artifacts/whl_dbfs/output.txt | 2 +- acceptance/bundle/artifacts/whl_dbfs/script | 2 +- .../bundle/artifacts/whl_dynamic/output.txt | 18 +-- .../bundle/artifacts/whl_dynamic/script | 4 +- .../bundle/artifacts/whl_explicit/output.txt | 6 +- .../bundle/artifacts/whl_explicit/script | 2 +- .../bundle/artifacts/whl_implicit/output.txt | 6 +- .../bundle/artifacts/whl_implicit/script | 2 +- .../whl_implicit_custom_path/output.txt | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../whl_implicit_notebook/output.txt | 6 +- .../artifacts/whl_implicit_notebook/script | 2 +- .../bundle/artifacts/whl_multiple/output.txt | 10 +- .../bundle/artifacts/whl_multiple/script | 2 +- .../artifacts/whl_no_cleanup/output.txt | 6 +- .../bundle/artifacts/whl_no_cleanup/script | 2 +- .../whl_prebuilt_multiple/output.txt | 10 +- .../artifacts/whl_prebuilt_multiple/script | 2 +- .../artifacts/whl_prebuilt_outside/output.txt | 6 +- .../artifacts/whl_prebuilt_outside/script | 2 +- .../whl_prebuilt_outside_dynamic/output.txt | 6 +- .../whl_prebuilt_outside_dynamic/script | 2 +- .../whl_via_environment_key/output.txt | 6 +- .../artifacts/whl_via_environment_key/script | 2 +- .../bind/pipelines/recreate/output.txt | 4 +- .../deployment/bind/pipelines/recreate/script | 6 +- .../bind/pipelines/update/output.txt | 2 +- .../deployment/bind/pipelines/update/script | 6 +- .../destroy/force-lock-node-limit/script | 4 +- .../outside_of_bundle_root/output.txt | 10 +- .../libraries/outside_of_bundle_root/script | 2 +- .../auto-migrate-empty-tfstate/output.txt | 24 +-- .../migrate/auto-migrate-empty-tfstate/script | 2 +- .../migrate/auto-migrate-push-failure/script | 17 +- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- .../change_assets_dir/output.txt | 2 +- .../quality_monitors/change_assets_dir/script | 2 +- .../change_output_schema_name/output.txt | 2 +- .../change_output_schema_name/script | 2 +- .../change_table_name/output.txt | 2 +- .../quality_monitors/change_table_name/script | 2 +- .../quality_monitors/create/output.txt | 2 +- .../resources/quality_monitors/create/script | 2 +- .../delete_scope/out.deploy.requests.txt | 2 +- .../secret_scopes/delete_scope/script | 2 +- .../sync-upload-edge-cases/databricks.yml | 2 + .../sync-upload-edge-cases/out.test.toml | 3 + .../bundle/sync-upload-edge-cases/output.txt | 140 ++++++++++++++++ .../bundle/sync-upload-edge-cases/script | 95 +++++++++++ .../bundle/sync-upload-edge-cases/test.toml | 18 +++ .../templates/default-python/classic/script | 4 +- .../default-python/serverless/script | 4 +- .../upload/internal_server_error/output.txt | 4 +- .../upload/internal_server_error/test.toml | 2 +- acceptance/bundle/upload/timeout/output.txt | 4 +- acceptance/bundle/upload/timeout/test.toml | 2 +- acceptance/bundle/user_agent/output.txt | 28 ++-- .../simple/out.requests.deploy.direct.json | 122 ++++++-------- .../simple/out.requests.deploy.terraform.json | 150 ++++++------------ .../simple/out.requests.destroy.direct.json | 20 ++- .../out.requests.destroy.terraform.json | 20 ++- .../validate/sync_patterns/out.sync.txt | 11 +- .../bundle/validate/sync_patterns/script | 2 +- .../cmd/sync/dryrun-missing-remote/output.txt | 4 +- libs/testserver/fault.go | 39 +++-- libs/testserver/fault_test.go | 48 +++--- libs/testserver/server.go | 2 +- 83 files changed, 682 insertions(+), 448 deletions(-) create mode 100644 .nextchanges/bundles/workspace-import-migration.md create mode 100644 acceptance/bundle/sync-upload-edge-cases/databricks.yml create mode 100644 acceptance/bundle/sync-upload-edge-cases/out.test.toml create mode 100644 acceptance/bundle/sync-upload-edge-cases/output.txt create mode 100644 acceptance/bundle/sync-upload-edge-cases/script create mode 100644 acceptance/bundle/sync-upload-edge-cases/test.toml diff --git a/.nextchanges/bundles/workspace-import-migration.md b/.nextchanges/bundles/workspace-import-migration.md new file mode 100644 index 00000000000..351faafb7cd --- /dev/null +++ b/.nextchanges/bundles/workspace-import-migration.md @@ -0,0 +1 @@ +Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a higher rate limit (30 vs 20 requests/sec per workspace) and is ~1.5–2× faster for typical bundle deployments. diff --git a/acceptance/bin/fault.py b/acceptance/bin/fault.py index 7d1ba207434..041008da1b7 100755 --- a/acceptance/bin/fault.py +++ b/acceptance/bin/fault.py @@ -1,15 +1,20 @@ #!/usr/bin/env python3 """Set up a fault rule on the testserver for the current test token. -Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] +Usage: fault.py [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE] - PATTERN HTTP method and path, supports trailing * wildcard, - e.g. "PUT /api/2.0/permissions/pipelines/*" - STATUS_CODE HTTP status code to return, e.g. 504 - OFFSET number of requests to let through before fault starts - TIMES number of times to return the fault response - ERROR_CODE optional error_code for the response body, e.g. - MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) + PATTERN HTTP method and path, supports trailing * wildcard, + e.g. "PUT /api/2.0/permissions/pipelines/*" + STATUS_CODE HTTP status code to return, e.g. 504 + OFFSET number of requests to let through before fault starts + TIMES number of times to return the fault response + ERROR_CODE optional error_code for the response body, e.g. + MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED) + --body-contains SUBSTR + only fire when the request body contains SUBSTR. Needed to + target a single file's /workspace/import upload, since every + upload shares the same method+path and only differs by the + multipart "path" form field. The rule is scoped to the current DATABRICKS_TOKEN so it only affects the test that registers it, even when tests share a server. @@ -27,12 +32,20 @@ print("DATABRICKS_HOST not set", file=sys.stderr) sys.exit(1) -if len(sys.argv) not in (5, 6): - print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr) +args = sys.argv[1:] +body_contains = "" +if len(args) >= 2 and args[0] == "--body-contains": + body_contains = args[1] + args = args[2:] + +if len(args) not in (4, 5): + print( + f"usage: {sys.argv[0]} [--body-contains SUBSTR] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr + ) sys.exit(1) -pattern, status_code, offset, times = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) -error_code = sys.argv[5] if len(sys.argv) == 6 else "INJECTED" +pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3]) +error_code = args[4] if len(args) == 5 else "INJECTED" body = json.dumps({"error_code": error_code, "message": "Fault injected by test."}) data = json.dumps( @@ -42,6 +55,7 @@ "body": body, "offset": offset, "times": times, + "body_contains": body_contains, } ).encode() diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt index 29cb7da70fa..e79c7a4f57c 100644 --- a/acceptance/bundle/apps/app_yaml/out.app.yml.txt +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -1,8 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "command:\n - python\n - app.py\n" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "command:\n - python\n - app.py\n", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml" + } + } } diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt index addb91d683c..7733b0f8ed5 100644 --- a/acceptance/bundle/apps/app_yaml/output.txt +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -19,7 +19,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq select(.path | test("app.yml")) out.requests.txt +>>> jq select(.body.multipart_form.path | strings | test("app.yml")) out.requests.txt >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script index d0dd8d5f800..adec87dc210 100644 --- a/acceptance/bundle/apps/app_yaml/script +++ b/acceptance/bundle/apps/app_yaml/script @@ -1,7 +1,7 @@ trace $CLI bundle validate trace $CLI bundle plan trace $CLI bundle deploy -trace jq 'select(.path | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt +trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt #trace print_requests.py //apps # currently fails due to TF inserting description="" rm out.requests.txt diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index a7ef4a0159f..4fb6f1c6dbb 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -24,5 +24,5 @@ Deployment complete! } === Expecting the code_source tarball to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" +>>> jq -r .body.multipart_form.path | strings +/Workspace/foo/bar/artifacts/.internal/code.tgz diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/script b/acceptance/bundle/artifacts/ai_runtime_code_source/script index 2b71cf524aa..8c7dcf45181 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/script +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/script @@ -4,6 +4,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' out.requests.txt title "Expecting the code_source tarball to be uploaded" -trace jq .path < out.requests.txt | grep import | grep code.tgz | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep code.tgz | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index dd81ba7de2a..b9d8cfeb72f 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -50,9 +50,9 @@ Deployment complete! ] === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script index e6b1437627c..cd28cc070f5 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/script +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/script @@ -8,7 +8,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index 6d24880e6c0..d309b10cad4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -6,9 +6,9 @@ Deploying resources... Deployment complete! === Expecting wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl +/Workspace/foo/bar/artifacts/.internal/source.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script index 883601185c9..b5ee485cbd0 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script @@ -5,7 +5,7 @@ echo "test wheel content" > whl/source.whl trace $CLI bundle deploy title "Expecting wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index fa725a29d86..7e5436900ed 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -40,15 +40,15 @@ Deployment complete! ] === Expecting 4 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl +/Workspace/foo/bar/artifacts/.internal/source1.whl +/Workspace/foo/bar/artifacts/.internal/source2.whl +/Workspace/foo/bar/artifacts/.internal/source3.whl +/Workspace/foo/bar/artifacts/.internal/source4.whl === Expecting environment dependencies to be updated >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/script b/acceptance/bundle/artifacts/upload_multiple_libraries/script index f615147f8c4..7625d8947f1 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/script +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/script @@ -11,7 +11,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 4 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting environment dependencies to be updated" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 60b60eff5d3..d7d1271b7d5 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -29,24 +29,24 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -67,16 +67,16 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -99,14 +99,14 @@ dist/my_test_code-0.2.0-py3-none-any.whl json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded ->>> jq .path out.requests.txt -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" +>>> jq -r .body.multipart_form.path | strings out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME diff --git a/acceptance/bundle/artifacts/whl_change_version/script b/acceptance/bundle/artifacts/whl_change_version/script index 9f6f480f0db..68f33c95633 100644 --- a/acceptance/bundle/artifacts/whl_change_version/script +++ b/acceptance/bundle/artifacts/whl_change_version/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt @@ -20,7 +20,7 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt @@ -34,6 +34,6 @@ title "Expecting 1 wheel in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 1 wheel to be uploaded" -trace jq .path out.requests.txt | grep import | sort +trace jq -r '.body.multipart_form.path | strings' out.requests.txt | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 306de22de1f..2d1d4ecbda0 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -27,6 +27,6 @@ Deployment complete! ] === Expecting no wheels to be uploaded ->>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl +>>> errcode sh -c jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index d7c93d8f38f..3f15b859126 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -7,6 +7,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting no wheels to be uploaded" -trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' +trace errcode sh -c "jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl" rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index c7409f37408..c66cee0294a 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -58,11 +58,11 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl === Updating the local wheel and deploying again Building my_test_code... @@ -94,7 +94,7 @@ json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 068db16caf8..1e02902b5e4 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -19,7 +19,7 @@ title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 2 patched wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt @@ -35,4 +35,4 @@ title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 6c1c06c46e0..bbac28dc117 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -29,9 +29,9 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting delete request to artifact_path/.internal folder >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 4988edd8155..db664434f00 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -6,7 +6,7 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort title "Expecting delete request to artifact_path/.internal folder" trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index 698cc20c514..a3dc47e7122 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -29,6 +29,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index da61fb91f83..948e73d11d9 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -6,6 +6,6 @@ title "Expecting 1 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 71f1bffc1df..dd3d94f7bec 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -41,6 +41,6 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl +/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index fdc0723f594..13f1ba1b6e9 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 7e393eabc47..1c5bbc12516 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -19,6 +19,6 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 30196556f1e..8d68a61e6da 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index a03025d9103..565f8524b95 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -35,8 +35,8 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 380c6c60f8a..dfaa1bbb94a 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -6,6 +6,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm -fr out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index a78810dbca9..b98f17dab65 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -11,9 +11,9 @@ Deployment complete! dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy Building python_artifact... diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/script b/acceptance/bundle/artifacts/whl_no_cleanup/script index 4536f929468..21e0d84ace8 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/script +++ b/acceptance/bundle/artifacts/whl_no_cleanup/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort trace $CLI bundle deploy title "No calls to delete internal folder expected" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index 4fe390e6350..02bf0311f7a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -38,8 +38,8 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 9df7ea3d987..977c9431eac 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -9,6 +9,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index 369db2e47a7..39495b82b18 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -32,6 +32,6 @@ Deployment complete! ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/script b/acceptance/bundle/artifacts/whl_prebuilt_outside/script index 939847f6c12..7459288248e 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/script @@ -11,6 +11,6 @@ title "Expecting 2 wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index 570b124a5bd..2615973a402 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -56,6 +56,6 @@ Deployment complete! ] === Expecting 2 wheels to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script index af4e84b8b3f..7306df94e9c 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/script @@ -14,6 +14,6 @@ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requ trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt title "Expecting 2 wheels to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index 8afa59e7d8b..a5468f4daef 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -49,6 +49,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded ->>> jq .path -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +>>> jq -r .body.multipart_form.path | strings +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl +/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 3a0dd929c09..0a4fbfcff24 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -6,6 +6,6 @@ title "Expecting 1 wheel in environments section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" -trace jq .path < out.requests.txt | grep import | grep whl | sort +trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort rm out.requests.txt diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index 4da2a167b7f..a15d1598098 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ +>>> print_requests.py ^//workspace/ >>> [CLI] bundle summary -o json @@ -33,4 +33,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index aba4cfb97a6..a6dcfbf5832 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -6,11 +6,11 @@ add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/' +print_requests.py '^//workspace/' rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' +trace print_requests.py '^//workspace/' trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan @@ -19,4 +19,4 @@ trace musterr $CLI bundle deploy rm -f out.requests.txt trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt index c1f096e648e..b2190526984 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -19,4 +19,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext +>>> print_requests.py ^//workspace/import ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 5d2e487f10c..3804b220e3d 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -3,15 +3,15 @@ add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//workspace/import' '^//workspace/delete' trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f out.requests.txt trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/import' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/destroy/force-lock-node-limit/script b/acceptance/bundle/destroy/force-lock-node-limit/script index c7fffbc16a2..616b626f2ef 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/script +++ b/acceptance/bundle/destroy/force-lock-node-limit/script @@ -7,9 +7,9 @@ title "Deploy (creates the job and the deployment)" trace $CLI bundle deploy title "destroy without --force-lock (lock write hits the node limit: aborts)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace errcode $CLI bundle destroy --auto-approve title "destroy with --force-lock (tolerates the node limit: proceeds and completes)" -fault.py "POST /api/2.0/workspace-files/import-file/*" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED +fault.py --body-contains "state/deploy.lock" "POST /api/2.0/workspace/import" 403 0 1 MAX_CHILD_NODE_SIZE_EXCEEDED trace $CLI bundle destroy --force-lock --auto-approve diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index e7d6530536a..85c2d65d202 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -53,8 +53,12 @@ Deployment complete! >>> cat out.requests.txt { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl" + } } } diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/script b/acceptance/bundle/libraries/outside_of_bundle_root/script index ff72be1b690..74f4f15cf87 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/script +++ b/acceptance/bundle/libraries/outside_of_bundle_root/script @@ -11,6 +11,6 @@ title "Check that the job libraries are uploaded and the path is correct in the trace cat out.requests.txt | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.tasks[0].libraries' trace cat out.requests.txt | jq 'select(.path == "/api/2.2/jobs/create")' | jq '.body.environments[0].spec.dependencies' trace cat out.requests.txt | jq 'select(.path == "/api/2.0/pipelines" and .method == "POST")' | jq '.body.environment.dependencies' -trace cat out.requests.txt | jq 'select(.path | test("/api/2.0/workspace-files/import-file/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl"))' +trace cat out.requests.txt | jq 'select(.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // "" | test("/Workspace/Users/.*/.bundle/outside_of_bundle_root/default/artifacts/.internal/test.whl")))' | jq 'del(.body.multipart_form.content)' rm out.requests.txt diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index c4dc9f4d8af..e19d0f5a81d 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -22,21 +22,21 @@ direct_migrated_via_config true === Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted ->>> print_requests.py //workspace-files/import-file //workspace/delete --sort -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" +>>> print_requests.py //workspace/import //workspace/delete --sort "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" +"POST /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" === debug states confirms no terraform.tfstate anywhere; only .backup remains diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script index 984bbf2fdce..afc0a262151 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script @@ -19,7 +19,7 @@ title "Sweep is recorded in telemetry as via-config\n" trace print_migration_telemetry title "Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted\n" -trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py 'terraform.tfstate.backup' 'workspace/delete' +trace print_requests.py //workspace/import //workspace/delete --sort | jq '.method + " " + (.body.multipart_form.path // .path)' | contains.py 'terraform.tfstate.backup' 'workspace/delete' title "debug states confirms no terraform.tfstate anywhere; only .backup remains\n" trace $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/script b/acceptance/bundle/migrate/auto-migrate-push-failure/script index b1ccb759da5..50df7951c7d 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/script +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/script @@ -1,5 +1,18 @@ export DATABRICKS_BUNDLE_ENGINE= +# Keep harness files out of the sync: their content (this script included) contains +# the literal "state/resources.json" that the fault below matches on, and a synced +# file carrying that string would trip the fault during the files phase instead of +# on the migration commit we mean to target. +cat > .gitignore <<'EOF' +script +output.txt +out.requests.txt +repls.json +test.toml +.databricks +EOF + title "Initial deploy on terraform" trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_migration_telemetry @@ -9,7 +22,7 @@ title "Opt in via env var; inject 403 on the resources.json push so commit fails # The auto-migration's commit path uploads resources.json to the workspace # before renaming any local files. A 403 (non-retryable) fails commitMigration → # the deploy prints a warning and leaves the local state as terraform. -fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 +fault.py --body-contains "state/resources.json" "POST /api/2.0/workspace/import" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f out.requests.txt @@ -27,4 +40,6 @@ title "Local state is now direct\n" trace find .databricks/bundle -name "resources.json" -type f trace find .databricks/bundle -name "terraform.tfstate*" -type f +rm -f .gitignore + rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..20a21048c32 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//workspace/import { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -29,7 +29,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//workspace/import { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort ^//import-file/ +>>> print_requests.py --sort ^//workspace/import { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index d38692366b3..51ce6274ed6 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' +trace print_requests.py '^//workspace/import' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' +trace print_requests.py '^//workspace/import' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' +trace print_requests.py --sort '^//workspace/import' diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 08f6c53ae17..b569000be3b 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 6caf49a7f4e..2bdc84ee12f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -26,5 +26,5 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index d67ee41975f..0f021b4f877 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index 72c57e08401..e7e800e526f 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index 879222aa8a4..48520444bbf 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 891aece1c11..0bec176f6a4 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -26,7 +26,7 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 8037d5ec9cc..7afde185783 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 78c7853b264..d14dee8fbef 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 2d469e4abce..b67ab287777 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index b12e98775a3..02a7ca6d1e8 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt trace $CLI bundle deploy -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/sync-upload-edge-cases/databricks.yml b/acceptance/bundle/sync-upload-edge-cases/databricks.yml new file mode 100644 index 00000000000..2261ff177c7 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: upload-edge-cases diff --git a/acceptance/bundle/sync-upload-edge-cases/out.test.toml b/acceptance/bundle/sync-upload-edge-cases/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/sync-upload-edge-cases/output.txt b/acceptance/bundle/sync-upload-edge-cases/output.txt new file mode 100644 index 00000000000..13e7506f2d6 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/output.txt @@ -0,0 +1,140 @@ + +>>> [CLI] bundle sync --output text +Action: PUT: .gitignore, dashboard.lvdash.json, databricks.yml, empty.txt, héllo.txt, large.bin, plain-script.py, pyNb.py, scalaNb.scala, sqlNb.sql, with spaces.txt +Initial Sync Complete +Uploaded .gitignore +Uploaded dashboard.lvdash.json +Uploaded databricks.yml +Uploaded empty.txt +Uploaded héllo.txt +Uploaded large.bin +Uploaded plain-script.py +Uploaded pyNb.py +Uploaded scalaNb.scala +Uploaded sqlNb.sql +Uploaded with spaces.txt + +=== workspace state after sync — assert object_type and language for each upload +>>> MSYS_NO_PATHCONV=1 [CLI] workspace list -o json /Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files +{ + "name": ".gitignore", + "object_type": "FILE", + "language": null +} +{ + "name": "dashboard.lvdash.json", + "object_type": "FILE", + "language": null +} +{ + "name": "databricks.yml", + "object_type": "FILE", + "language": null +} +{ + "name": "empty.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "héllo.txt", + "object_type": "FILE", + "language": null +} +{ + "name": "large.bin", + "object_type": "FILE", + "language": null +} +{ + "name": "plain-script.py", + "object_type": "FILE", + "language": null +} +{ + "name": "pyNb", + "object_type": "NOTEBOOK", + "language": "PYTHON" +} +{ + "name": "scalaNb", + "object_type": "NOTEBOOK", + "language": "SCALA" +} +{ + "name": "sqlNb", + "object_type": "NOTEBOOK", + "language": "SQL" +} +{ + "name": "with spaces.txt", + "object_type": "FILE", + "language": null +} + +=== uploaded paths from each multipart POST /workspace/import (sorted) +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.path) | + sort | .[] + out.requests.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/.gitignore +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/dashboard.lvdash.json +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/databricks.yml +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/empty.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/héllo.txt +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/large.bin +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/plain-script.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/pyNb.py +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/scalaNb.scala +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/sqlNb.sql +/Workspace/Users/[USERNAME]/.bundle/upload-edge-cases/default/files/with spaces.txt + +=== every upload set format=AUTO +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique + out.requests.txt +[ + "AUTO" +] + +=== recorded content for the small text inputs +>>> jq -rs + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] + out.requests.txt +{ + "path": "dashboard.lvdash.json", + "content": "{\"datasets\":[],\"pages\":[]}\n" +} +{ + "path": "empty.txt", + "content": "" +} +{ + "path": "héllo.txt", + "content": "hello, naïve world\n" +} +{ + "path": "plain-script.py", + "content": "print(\"plain script, no header\")\n" +} +{ + "path": "pyNb.py", + "content": "# Databricks notebook source\nprint(\"hello\")\n" +} +{ + "path": "scalaNb.scala", + "content": "// Databricks notebook source\nprintln(\"hello\")\n" +} +{ + "path": "sqlNb.sql", + "content": "-- Databricks notebook source\nSELECT 1\n" +} +{ + "path": "with spaces.txt", + "content": "hello, world\n" +} diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script new file mode 100644 index 00000000000..b0402a2b003 --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -0,0 +1,95 @@ +# Confidence checks that /workspace/import is a drop-in for the legacy import-file: +# verify the CLI's multipart upload pipeline handles the cases that differ between the +# two endpoints — large content (>10 MiB), empty content, varied extensions/object types, +# and non-ASCII / spaced filenames in the multipart `path` field. + +# Exclude framework-generated and script-internal files from the sync. We only want the +# inputs we explicitly generate below to show up in the recorded uploads. +cat > .gitignore <<'EOF' +script +output.txt +out.requests.txt +repls.json +test.toml +.databricks +EOF + +# 12 MiB binary file. The legacy /workspace/import JSON-body cap is 10 MiB; multipart has +# no such cap. Generated dynamically to keep the repo small. +python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" + +# Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). +touch empty.txt + +# Python notebook: auto-detected as NOTEBOOK by /workspace/import; testserver mirrors +# this and stores the object at the path with .py stripped. +cat > pyNb.py <<'EOF' +# Databricks notebook source +print("hello") +EOF + +# SQL notebook: -- comment header, .sql extension, Language=SQL on the workspace side. +cat > sqlNb.sql <<'EOF' +-- Databricks notebook source +SELECT 1 +EOF + +# Scala notebook: // comment header, .scala extension, Language=SCALA. +cat > scalaNb.scala <<'EOF' +// Databricks notebook source +println("hello") +EOF + +# Plain .py without the source header — should land as FILE, not NOTEBOOK, +# even though the extension is .py. +cat > plain-script.py <<'EOF' +print("plain script, no header") +EOF + +# Lakeview dashboard descriptor: the real /workspace/import assigns object_type=DASHBOARD +# and preserves the .lvdash.json extension. The testserver doesn't emulate that and stores +# it as a generic FILE — but the upload-side request shape (which is what this test asserts) +# is identical. +echo '{"datasets":[],"pages":[]}' > dashboard.lvdash.json + +# Non-ASCII filename. Multipart encodes filenames with RFC 5987 / quoted-string rules, +# distinct from URL-encoding in the legacy import-file endpoint. +echo "hello, naïve world" > "héllo.txt" + +# Filename with a space. +echo "hello, world" > "with spaces.txt" + +trace $CLI bundle sync --output text 2>&1 | sort + +title "workspace state after sync — assert object_type and language for each upload" +remote_files=$($CLI bundle summary -o json 2>/dev/null | jq -r .workspace.file_path) +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-slash +# workspace path into a C:/Program Files/... filesystem path before it +# reaches the CLI. +trace MSYS_NO_PATHCONV=1 $CLI workspace list -o json "$remote_files" | + jq 'sort_by(.path) | .[] | {name: (.path | sub(".*/"; "")), object_type, language}' + +title "uploaded paths from each multipart POST /workspace/import (sorted)" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.path) | + sort | .[] +' out.requests.txt + +title "every upload set format=AUTO" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(.body.multipart_form.format) | unique +' out.requests.txt + +title "recorded content for the small text inputs" +trace jq -rs ' + map(select(.path == "/api/2.0/workspace/import")) | + map(select(.body.multipart_form.path | test("/(pyNb\\.py|sqlNb\\.sql|scalaNb\\.scala|plain-script\\.py|héllo\\.txt|with spaces\\.txt|empty\\.txt|dashboard\\.lvdash\\.json)$"))) | + map({path: (.body.multipart_form.path | sub(".*/"; "")), content: .body.multipart_form.content}) | + sort_by(.path) | .[] +' out.requests.txt + +# Drop the recorded requests; their workspace-prefixed paths are noisy and the +# assertions above already pin the relevant fields. +rm out.requests.txt .gitignore diff --git a/acceptance/bundle/sync-upload-edge-cases/test.toml b/acceptance/bundle/sync-upload-edge-cases/test.toml new file mode 100644 index 00000000000..8bfde26201e --- /dev/null +++ b/acceptance/bundle/sync-upload-edge-cases/test.toml @@ -0,0 +1,18 @@ +RecordRequests = true + +# All sync uploads go through the same /workspace/import multipart code path; running +# both engine variants is unnecessary here. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "large.bin", + "empty.txt", + "pyNb.py", + "sqlNb.sql", + "scalaNb.scala", + "plain-script.py", + "dashboard.lvdash.json", + "héllo.txt", + "with spaces.txt", + ".databricks", +] diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 6b79c6f2251..1cf42e037e4 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,12 +13,12 @@ $CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 86272b4a266..12d51337bdf 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -12,12 +12,12 @@ $CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE rm ../../out.requests.txt trace $CLI bundle deploy -t dev -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod -print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py --sort '^//workspace/import' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/upload/internal_server_error/output.txt b/acceptance/bundle/upload/internal_server_error/output.txt index 5cde6aad650..24d8b2cc4a5 100644 --- a/acceptance/bundle/upload/internal_server_error/output.txt +++ b/acceptance/bundle/upload/internal_server_error/output.txt @@ -1,7 +1,7 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: Failed to acquire deployment lock: Internal Server Error Error: Internal Server Error (500) -Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace/import HTTP Status: 500 Internal Server Error API error_code: API message: Internal Server Error diff --git a/acceptance/bundle/upload/internal_server_error/test.toml b/acceptance/bundle/upload/internal_server_error/test.toml index 8ff8501b6de..b1d2b7dc49f 100644 --- a/acceptance/bundle/upload/internal_server_error/test.toml +++ b/acceptance/bundle/upload/internal_server_error/test.toml @@ -1,3 +1,3 @@ [[Server]] -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 500 diff --git a/acceptance/bundle/upload/timeout/output.txt b/acceptance/bundle/upload/timeout/output.txt index c314a247707..3b6eb72f28b 100644 --- a/acceptance/bundle/upload/timeout/output.txt +++ b/acceptance/bundle/upload/timeout/output.txt @@ -1,3 +1,3 @@ -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: Post "[DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Ffiles%2Ffile_to_upload.txt?overwrite=true": request timed out after 5s of inactivity +Error: Failed to acquire deployment lock: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity +Error: Post "[DATABRICKS_URL]/api/2.0/workspace/import": request timed out after 5s of inactivity diff --git a/acceptance/bundle/upload/timeout/test.toml b/acceptance/bundle/upload/timeout/test.toml index de2d8ce015b..ed84a5d1db8 100644 --- a/acceptance/bundle/upload/timeout/test.toml +++ b/acceptance/bundle/upload/timeout/test.toml @@ -4,5 +4,5 @@ DATABRICKS_BUNDLE_HTTP_TIMEOUT_SECONDS = "5" [[Server]] # CLI aborts after a single attempt when the HTTP timeout fires. Delay = "30s" -Pattern = "POST /api/2.0/workspace-files/import-file/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/files/file_to_upload.txt" +Pattern = "POST /api/2.0/workspace/import" Response.StatusCode = 200 diff --git a/acceptance/bundle/user_agent/output.txt b/acceptance/bundle/user_agent/output.txt index 3fe73103691..24c1c347f27 100644 --- a/acceptance/bundle/user_agent/output.txt +++ b/acceptance/bundle/user_agent/output.txt @@ -8,14 +8,14 @@ OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct OK deploy.direct /api/2.0/workspace/get-status engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/direct -OK deploy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct OK deploy.direct /api/2.0/workspace/delete engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct +OK deploy.direct /api/2.0/workspace/import engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.0/workspace/mkdirs engine/direct OK deploy.direct /api/2.1/unity-catalog/schemas engine/direct @@ -30,14 +30,14 @@ OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform OK deploy.terraform /api/2.0/workspace/get-status engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json engine/terraform -OK deploy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate engine/terraform OK deploy.terraform /api/2.0/workspace/delete engine/terraform OK deploy.terraform /api/2.0/workspace/delete engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform +OK deploy.terraform /api/2.0/workspace/import engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform OK deploy.terraform /api/2.0/workspace/mkdirs engine/terraform MISS deploy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' @@ -58,8 +58,8 @@ OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.0/workspace/get-status engine/direct OK destroy.direct /api/2.1/unity-catalog/schemas/mycatalog.myschema engine/direct -OK destroy.direct /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/direct OK destroy.direct /api/2.0/workspace/delete engine/direct +OK destroy.direct /api/2.0/workspace/import engine/direct MISS destroy.direct /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' MISS destroy.terraform /api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_destroy cmd-exec-id/[UUID] interactive/none auth/pat' @@ -69,8 +69,8 @@ OK destroy.terraform /api/2.0/workspace/export engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform OK destroy.terraform /api/2.0/workspace/get-status engine/terraform -OK destroy.terraform /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock engine/terraform OK destroy.terraform /api/2.0/workspace/delete engine/terraform +OK destroy.terraform /api/2.0/workspace/import engine/terraform MISS destroy.terraform /.well-known/databricks-config 'cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]' MISS destroy.terraform /api/2.1/unity-catalog/schemas/mycatalog.myschema 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' MISS destroy.terraform /api/2.0/preview/scim/v2/Me 'databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat' diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 5326f079064..c706828fa18 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -130,9 +130,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -142,15 +143,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { @@ -160,15 +155,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/import", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" + } } } { @@ -178,22 +172,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "seq": 1, - "cli_version": "[CLI_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "empty.py", - "is_notebook": false - } - ], - "id": "[UUID]" + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -203,29 +188,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "config": { - "bundle": { - "name": "test-bundle", - "target": "default", - "git": { - "bundle_root_path": "." - } - }, - "workspace": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" - }, - "resources": {}, - "presets": { - "source_linked_deployment": false - } - }, - "extra": {} + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -235,23 +204,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "state": { - "resources.schemas.foo": { - "__id__": "mycatalog.myschema", - "state": { - "catalog_name": "mycatalog", - "name": "myschema" - } - } + "multipart_form": { + "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" } } } @@ -262,10 +221,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "multipart_form": { + "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + } } } { @@ -275,9 +238,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "multipart_form": { + "content": "{\n \"state_version\": 2,\n \"cli_version\": \"[CLI_VERSION]\",\n \"lineage\": \"[UUID]\",\n \"serial\": 1,\n \"state\": {\n \"resources.schemas.foo\": {\n \"__id__\": \"mycatalog.myschema\",\n \"state\": {\n \"catalog_name\": \"mycatalog\",\n \"name\": \"myschema\"\n }\n }\n }\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json index 9bfe34ca4dd..d7d9edd978d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.terraform.json @@ -130,9 +130,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", + "recursive": true } } { @@ -142,15 +143,9 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" } } { @@ -160,15 +155,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/import", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/empty.py" + } } } { @@ -178,22 +172,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "seq": 1, - "cli_version": "[CLI_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "empty.py", - "is_notebook": false - } - ], - "id": "[UUID]" + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -203,29 +188,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 1, - "config": { - "bundle": { - "name": "test-bundle", - "target": "default", - "git": { - "bundle_root_path": "." - } - }, - "workspace": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" - }, - "resources": {}, - "presets": { - "source_linked_deployment": false - } - }, - "extra": {} + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { @@ -235,50 +204,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate", - "q": { - "overwrite": "true" - }, + "path": "/api/2.0/workspace/import", "body": { - "version": 4, - "terraform_version": "1.5.5", - "serial": 1, - "lineage": "[UUID]", - "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "databricks_schema", - "name": "foo", - "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "catalog_name": "mycatalog", - "comment": null, - "enable_predictive_optimization": "INHERIT", - "force_destroy": true, - "id": "mycatalog.myschema", - "metastore_id": "[UUID]", - "name": "myschema", - "owner": "[USERNAME]", - "properties": null, - "provider_config": [ - { - "workspace_id": "[NUMID]" - } - ], - "schema_id": "[UUID]", - "storage_root": null - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - } - ], - "check_results": null + "multipart_form": { + "content": "{\"version\":1,\"seq\":1,\"cli_version\":\"[CLI_VERSION]\",\"timestamp\":\"[TIMESTAMP]\",\"files\":[{\"local_path\":\"empty.py\",\"is_notebook\":false}],\"id\":\"[UUID]\"}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" + } } } { @@ -288,10 +221,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal", - "recursive": true + "multipart_form": { + "content": "{\n \"version\": 1,\n \"config\": {\n \"bundle\": {\n \"name\": \"test-bundle\",\n \"target\": \"default\",\n \"git\": {\n \"bundle_root_path\": \".\"\n }\n },\n \"workspace\": {\n \"file_path\": \"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"\n },\n \"resources\": {},\n \"presets\": {\n \"source_linked_deployment\": false\n }\n },\n \"extra\": {}\n}", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + } } } { @@ -301,9 +238,14 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + "multipart_form": { + "content": "{\n \"version\": 4,\n \"terraform_version\": \"1.5.5\",\n \"serial\": 1,\n \"lineage\": \"[UUID]\",\n \"outputs\": {},\n \"resources\": [\n {\n \"mode\": \"managed\",\n \"type\": \"databricks_schema\",\n \"name\": \"foo\",\n \"provider\": \"provider[\\\"registry.terraform.io/databricks/databricks\\\"]\",\n \"instances\": [\n {\n \"schema_version\": 0,\n \"attributes\": {\n \"catalog_name\": \"mycatalog\",\n \"comment\": null,\n \"enable_predictive_optimization\": \"INHERIT\",\n \"force_destroy\": true,\n \"id\": \"mycatalog.myschema\",\n \"metastore_id\": \"[UUID]\",\n \"name\": \"myschema\",\n \"owner\": \"[USERNAME]\",\n \"properties\": null,\n \"provider_config\": [\n {\n \"workspace_id\": \"[NUMID]\"\n }\n ],\n \"schema_id\": \"[UUID]\",\n \"storage_root\": null\n },\n \"sensitive_attributes\": [],\n \"private\": \"bnVsbA==\"\n }\n ]\n }\n ],\n \"check_results\": null\n}\n", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json index c67f22e3730..e22956fe86d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.direct.json @@ -121,15 +121,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { @@ -139,10 +134,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { diff --git a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json index a8119cf870d..599329568a3 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json +++ b/acceptance/bundle/user_agent/simple/out.requests.destroy.terraform.json @@ -100,15 +100,10 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, + "path": "/api/2.0/workspace/delete", "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", + "recursive": true } } { @@ -118,10 +113,13 @@ ] }, "method": "POST", - "path": "/api/2.0/workspace/delete", + "path": "/api/2.0/workspace/import", "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default", - "recursive": true + "multipart_form": { + "content": "{\"ID\":\"[UUID]\",\"AcquisitionTime\":\"[TIMESTAMP]\",\"IsForced\":false,\"User\":\"[USERNAME]\"}", + "format": "AUTO", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" + } } } { diff --git a/acceptance/bundle/validate/sync_patterns/out.sync.txt b/acceptance/bundle/validate/sync_patterns/out.sync.txt index ac5aba9ab29..fd939c9721b 100644 --- a/acceptance/bundle/validate/sync_patterns/out.sync.txt +++ b/acceptance/bundle/validate/sync_patterns/out.sync.txt @@ -1,7 +1,12 @@ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/conf/dir/test.yml" + } } } diff --git a/acceptance/bundle/validate/sync_patterns/script b/acceptance/bundle/validate/sync_patterns/script index d2aae85444a..ce8e5cbb1b6 100644 --- a/acceptance/bundle/validate/sync_patterns/script +++ b/acceptance/bundle/validate/sync_patterns/script @@ -1,5 +1,5 @@ trace $CLI bundle validate trace $CLI bundle validate -o json | jq '.sync' trace $CLI bundle deploy -jq 'select(.path | test("dir/test.yml"))' out.requests.txt > out.sync.txt +jq 'select(.body.multipart_form.path | strings | test("dir/test.yml"))' out.requests.txt > out.sync.txt rm out.requests.txt diff --git a/acceptance/cmd/sync/dryrun-missing-remote/output.txt b/acceptance/cmd/sync/dryrun-missing-remote/output.txt index 4848aae1c2d..b7453994b58 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/output.txt +++ b/acceptance/cmd/sync/dryrun-missing-remote/output.txt @@ -15,11 +15,11 @@ Uploaded hello.py >>> print_requests.py --sort //api/ { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/.gitignore" + "path": "/api/2.0/workspace/import" } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Users/[USERNAME]/missing-dir-[UNIQUE_NAME]/hello.py" + "path": "/api/2.0/workspace/import" } { "method": "POST", diff --git a/libs/testserver/fault.go b/libs/testserver/fault.go index 49b98b0f25f..47cde420ee9 100644 --- a/libs/testserver/fault.go +++ b/libs/testserver/fault.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "strings" "sync" @@ -17,6 +18,12 @@ type FaultRule struct { Body string offset int times int + + // bodyContains, if non-empty, additionally requires the request body to + // contain this substring for the rule to fire. /workspace/import routes + // every upload through the same method+path, so the only way to target a + // single file's upload is by matching its multipart "path" form field. + bodyContains string } // FaultRules holds the active fault injection rules for a test server. @@ -31,20 +38,22 @@ func NewFaultRules() *FaultRules { } // Set registers or replaces a fault rule for the given token and pattern. -func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int) { +func (fr *FaultRules) Set(token, pattern string, statusCode int, body string, offset, times int, bodyContains string) { fr.mu.Lock() defer fr.mu.Unlock() fr.rules[faultRuleKey{token: token, pattern: pattern}] = &FaultRule{ - StatusCode: statusCode, - Body: body, - offset: offset, - times: times, + StatusCode: statusCode, + Body: body, + offset: offset, + times: times, + bodyContains: bodyContains, } } // Check returns a matching fault rule and advances its counters, or nil if no rule matches. // Pattern supports a trailing "*" wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*". -func (fr *FaultRules) Check(method, path, token string) *FaultRule { +// A rule with a non-empty bodyContains only matches when body contains that substring. +func (fr *FaultRules) Check(method, path, token string, body []byte) *FaultRule { requestPattern := method + " " + path fr.mu.Lock() @@ -64,6 +73,11 @@ func (fr *FaultRules) Check(method, path, token string) *FaultRule { if !matched { continue } + // A non-matching body must not consume the rule's offset/times budget, + // so this check precedes the counter updates below. + if rule.bodyContains != "" && !bytes.Contains(body, []byte(rule.bodyContains)) { + continue + } if rule.offset > 0 { rule.offset-- return nil @@ -87,16 +101,17 @@ func (fr *FaultRules) Check(method, path, token string) *FaultRule { func faultEndpointHandler(fr *FaultRules) HandlerFunc { return func(req Request) any { var body struct { - Pattern string `json:"pattern"` - StatusCode int `json:"status_code"` - Body string `json:"body"` - Offset int `json:"offset"` - Times int `json:"times"` + Pattern string `json:"pattern"` + StatusCode int `json:"status_code"` + Body string `json:"body"` + Offset int `json:"offset"` + Times int `json:"times"` + BodyContains string `json:"body_contains"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: 400, Body: map[string]string{"error": err.Error()}} } - fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times) + fr.Set(req.Token, body.Pattern, body.StatusCode, body.Body, body.Offset, body.Times, body.BodyContains) return Response{StatusCode: 200} } } diff --git a/libs/testserver/fault_test.go b/libs/testserver/fault_test.go index 2366fb733c1..ac6b7ce58a3 100644 --- a/libs/testserver/fault_test.go +++ b/libs/testserver/fault_test.go @@ -10,18 +10,18 @@ import ( func TestFaultRulesNoMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 1) + fr.Set("tok", "GET /foo", 504, "body", 0, 1, "") - assert.Nil(t, fr.Check("POST", "/foo", "tok")) - assert.Nil(t, fr.Check("GET", "/bar", "tok")) - assert.Nil(t, fr.Check("GET", "/foo", "other")) + assert.Nil(t, fr.Check("POST", "/foo", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/bar", "tok", nil)) + assert.Nil(t, fr.Check("GET", "/foo", "other", nil)) } func TestFaultRulesExactMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1) + fr.Set("tok", "PUT /api/2.0/jobs/123", 504, "body", 0, 1, "") - rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok") + rule := fr.Check("PUT", "/api/2.0/jobs/123", "tok", nil) require.NotNil(t, rule) assert.Equal(t, 504, rule.StatusCode) assert.Equal(t, "body", rule.Body) @@ -29,29 +29,41 @@ func TestFaultRulesExactMatch(t *testing.T) { func TestFaultRulesWildcardMatch(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2) + fr.Set("tok", "PUT /api/2.0/permissions/pipelines/*", 504, "body", 0, 2, "") - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok")) - assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) - assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok")) // exhausted + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/abc", "tok", nil)) + assert.NotNil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) + assert.Nil(t, fr.Check("PUT", "/api/2.0/permissions/pipelines/xyz", "tok", nil)) // exhausted } func TestFaultRulesOffset(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 2, 1) + fr.Set("tok", "GET /foo", 504, "body", 2, 1, "") - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 2→1 - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // offset 1→0 - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) // fires - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 2→1 + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // offset 1→0 + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) // fires + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted } func TestFaultRulesTimes(t *testing.T) { fr := testserver.NewFaultRules() - fr.Set("tok", "GET /foo", 504, "body", 0, 3) + fr.Set("tok", "GET /foo", 504, "body", 0, 3, "") for range 3 { - assert.NotNil(t, fr.Check("GET", "/foo", "tok")) + assert.NotNil(t, fr.Check("GET", "/foo", "tok", nil)) } - assert.Nil(t, fr.Check("GET", "/foo", "tok")) // exhausted + assert.Nil(t, fr.Check("GET", "/foo", "tok", nil)) // exhausted +} + +func TestFaultRulesBodyContains(t *testing.T) { + fr := testserver.NewFaultRules() + fr.Set("tok", "POST /api/2.0/workspace/import", 403, "body", 0, 1, "state/resources.json") + + // Body without the substring must not match and must not consume the budget. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/deploy.lock"))) + // Body with the substring fires. + assert.NotNil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) + // Budget was only consumed by the matching request. + assert.Nil(t, fr.Check("POST", "/api/2.0/workspace/import", "tok", []byte("path=state/resources.json"))) } diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5ae3141bfd2..2bd67f20c21 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, handler HandlerFu var resp EncodedResponse - if rule := s.faults.Check(r.Method, r.URL.Path, token); rule != nil { + if rule := s.faults.Check(r.Method, r.URL.Path, token, request.Body); rule != nil { resp = EncodedResponse{ StatusCode: rule.StatusCode, Body: []byte(rule.Body), From 97a20ed3643ab1f6d23bb63589ad3157dc97eaa8 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 4 Aug 2026 11:27:52 +0000 Subject: [PATCH 3/5] update comments --- libs/filer/workspace_files_client.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 5b903ab3678..1f6c21571e0 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -239,6 +239,9 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // targets /foo.py: AUTO would store the new content as FILE at // /foo.py, but the workspace treats /foo.py as the source view of // the existing /foo NOTEBOOK and rejects the type change. + // Unlike (a), this message comes from the legacy webapp tree path + // (webapp/.../tree/TreeBackendHelper.scala), not from WCS, so the + // WCS-only ErrorInfo work below does not cover it. // // The server refuses the overwrite even though the caller asked for // it; from the caller's perspective the path is occupied, so we @@ -254,10 +257,19 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io if info := aerr.ErrorDetails().ErrorInfo; info != nil && info.Reason == workspaceObjectTypeMismatchReason { return fileAlreadyExistsError{absPath} } - // Fallback for workspaces where the ErrorInfo change has not - // rolled out: as of 2026-06-12 aws-prod-ucws still returns these - // errors without details, so match the two observed messages. - // Remove once the rollout is confirmed everywhere. + // Fallback for errors that carry no ErrorInfo. Two reasons this + // is still needed, both verified against a live workspace on + // 2026-08-04: + // + // - Rollout lag: universe #2019174 merged 2026-06-03 with its + // SAFE flag defaulting to true, but workspaces on an older WCS + // build still return WCS-worded collisions without details. + // That half is temporary. + // + // - Message (b) above is thrown by webapp, which #2019174 never + // touched, so it has no ErrorInfo regardless of WCS rollout. + // Once the lag clears this can narrow to "node type" alone, + // but it cannot be dropped until webapp attaches details too. if strings.Contains(aerr.Message, "type mismatch") || strings.Contains(aerr.Message, "node type") { return fileAlreadyExistsError{absPath} } From 781d25139ee26dfda39f0a29d469c6d9ff792827 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 5 Aug 2026 13:14:57 +0000 Subject: [PATCH 4/5] Source upload size and rate limits from public docs The size limits in WorkspaceFilesClient.Write were described using internal config flag names and figures measured by hand against one workspace, neither of which a reader can verify. Replace them with the documented limits and link the sources. This corrects one figure: the notebook cap was described as a flat 10 MiB, but IPYNB notebooks are documented at 100 MB. Only source-format notebooks are capped at 10 MB, so the limit depends on how format=AUTO classifies the payload. Also restore the rate limit doc link to MaxRequestsInFlight, which the comment carried when the constant was introduced in #81. The value is unchanged. Add a comment above each test in workspace_files_client_test.go stating what it covers. Co-authored-by: Isaac --- .../bundles/workspace-import-migration.md | 2 +- .../bundle/sync-upload-edge-cases/script | 7 ++-- libs/filer/workspace_files_client.go | 38 ++++++++++++------- libs/filer/workspace_files_client_test.go | 30 +++++++++++++++ libs/sync/watchdog.go | 5 ++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.nextchanges/bundles/workspace-import-migration.md b/.nextchanges/bundles/workspace-import-migration.md index 351faafb7cd..4212003bdc7 100644 --- a/.nextchanges/bundles/workspace-import-migration.md +++ b/.nextchanges/bundles/workspace-import-migration.md @@ -1 +1 @@ -Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a higher rate limit (30 vs 20 requests/sec per workspace) and is ~1.5–2× faster for typical bundle deployments. +Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a documented rate limit of 30 requests per second per workspace ([API rate limits](https://docs.databricks.com/aws/en/resources/limits)), higher than the limit that applied to the previous endpoint, and is ~1.5–2× faster for typical bundle deployments. diff --git a/acceptance/bundle/sync-upload-edge-cases/script b/acceptance/bundle/sync-upload-edge-cases/script index b0402a2b003..950c2157f3c 100644 --- a/acceptance/bundle/sync-upload-edge-cases/script +++ b/acceptance/bundle/sync-upload-edge-cases/script @@ -1,6 +1,6 @@ # Confidence checks that /workspace/import is a drop-in for the legacy import-file: # verify the CLI's multipart upload pipeline handles the cases that differ between the -# two endpoints — large content (>10 MiB), empty content, varied extensions/object types, +# two endpoints — large content (>10 MB), empty content, varied extensions/object types, # and non-ASCII / spaced filenames in the multipart `path` field. # Exclude framework-generated and script-internal files from the sync. We only want the @@ -14,8 +14,9 @@ test.toml .databricks EOF -# 12 MiB binary file. The legacy /workspace/import JSON-body cap is 10 MiB; multipart has -# no such cap. Generated dynamically to keep the repo small. +# 12 MiB binary file, over the 10 MB cap on the JSON-body variant's base64 `content` +# field. The multipart variant this test exercises has no such cap. Generated +# dynamically to keep the repo small. python3 -c "open('large.bin', 'wb').write(b'\\0' * (12 * 1024 * 1024))" # Empty file (multipart encodes an empty `content` part — distinct from JSON's empty string). diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 1f6c21571e0..24714339119 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -164,20 +164,32 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - // Use Workspace.Upload (multipart /api/2.0/workspace/import) instead of the - // JSON-body variant of the same endpoint, which caps payloads at 10 MiB for - // AUTO format (databricks.webapp.autoExportFormatLimitBytes). The multipart - // variant has been verified against a real workspace at 450 MB for regular - // files — strictly better than the previous /workspace-files/import-file - // endpoint, which has a 200 MiB body cap - // (databricks.workspaceFilesystem.maxImportSizeBytes) plus a 305s server-side - // request timeout that cuts off uploads above ~400 MB at typical bandwidth. + // Use Workspace.Upload (multipart /api/2.0/workspace/import) rather than + // Workspace.Import (the JSON-body variant of the same endpoint). The JSON + // variant sends content base64-encoded in a `content` field that is capped + // at 10 MB, returning MAX_NOTEBOOK_SIZE_EXCEEDED above it; the multipart + // variant posts the bytes as a file part instead and is bounded only by the + // 500 MB workspace file size limit. See the `content` field description in + // .codegen/cli.json (workspace.Import) and + // https://docs.databricks.com/aws/en/files/workspace ("Workspace file size + // is limited to 500MB"). // - // Notebook content (any payload with a `# Databricks notebook source` header - // detected by format=AUTO) hits a separate 10 MiB cap on the server - // (databricks.notebook.maxNotebookSizeBytes). Both /workspace-files/import-file - // and /workspace/import enforce this same cap, so switching from the former - // to the latter does not regress maximum notebook upload size. + // Because format=AUTO lets the server classify each payload, the applicable + // limit depends on how the content is classified, not on our request: + // + // - Regular file: 500 MB. + // - Notebook in a source format, i.e. content whose extension and + // "Databricks notebook source" header make AUTO import it as a + // notebook: 10 MB. + // - Notebook in IPYNB format: 100 MB. + // + // Notebook limits are documented at + // https://docs.databricks.com/aws/en/notebooks/notebook-limitations + // ("Import and export is supported for IPYNB notebooks up to 100 MB" and up + // to 10 MB for other formats). They are enforced by the workspace for both + // /workspace/import and the /workspace-files/import-file endpoint this + // replaced, so migrating between the two does not change the maximum + // uploadable notebook size. overwrite := slices.Contains(mode, OverwriteIfExists) uploadOpts := []func(*workspace.Import){ workspace.UploadFormat(workspace.ImportFormatAuto), diff --git a/libs/filer/workspace_files_client_test.go b/libs/filer/workspace_files_client_test.go index a607b116263..2b71eb9820d 100644 --- a/libs/filer/workspace_files_client_test.go +++ b/libs/filer/workspace_files_client_test.go @@ -22,6 +22,8 @@ import ( "github.com/stretchr/testify/require" ) +// Confirms directory entries derived from workspace object infos report the +// name, type, and size of each object type the workspace can return. func TestWorkspaceFilesDirEntry(t *testing.T) { entries := wsfsDirEntriesFromObjectInfos([]workspace.ObjectInfo{ { @@ -68,6 +70,8 @@ func TestWorkspaceFilesDirEntry(t *testing.T) { assert.True(t, i2.IsDir()) } +// Confirms the workspace ID routing header is set only when the client config +// carries a workspace ID, and that a nil workspace client yields no headers. func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { tests := []struct { name string @@ -104,6 +108,11 @@ func TestWorkspaceFilesClientWorkspaceIDHeaders(t *testing.T) { }) } +// Confirms Write uploads through Workspace.Upload with format=AUTO, and that +// the overwrite flag is set only when OverwriteIfExists is passed. AUTO is what +// lets the server decide between storing content as a file or a notebook, so +// the format is asserted explicitly rather than left to the SDK default +// (SOURCE), which would import every file as a notebook. func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { tests := []struct { name string @@ -157,6 +166,11 @@ func TestWorkspaceFilesClientWriteSuccess(t *testing.T) { } } +// Confirms each error /workspace/import returns is translated to the filer's +// own error type, and that unrecognized errors pass through unchanged. The +// endpoint signals an occupied path with three different status/error_code +// combinations (see the comment on Write), so all three are covered here to +// pin the mapping the rest of the CLI relies on. func TestWorkspaceFilesClientWriteErrorMapping(t *testing.T) { tests := []struct { name string @@ -287,6 +301,8 @@ func writeWithImportError(t *testing.T, body map[string]any) error { return err } +// Confirms a type-mismatch collision is recognized from the structured +// AIP-193 ErrorInfo reason, independent of the error message wording. func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { // The message is deliberately one the fallback string match does not // recognize, to prove the branch fires on the structured reason alone. @@ -306,6 +322,8 @@ func TestWorkspaceFilesClientWriteTypeMismatchReason(t *testing.T) { assert.ErrorAs(t, err, &target) } +// Confirms an ErrorInfo carrying some other reason is not mistaken for a path +// collision, so the reason check above cannot swallow unrelated failures. func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { err := writeWithImportError(t, map[string]any{ "error_code": "INVALID_PARAMETER_VALUE", @@ -323,6 +341,10 @@ func TestWorkspaceFilesClientWriteUnrelatedReasonPassesThrough(t *testing.T) { assert.Equal(t, http.StatusBadRequest, aerr.StatusCode) } +// Confirms a 404 from a missing parent directory triggers a mkdirs call and a +// retry of the upload when CreateParentDirectories is passed. The retry re-reads +// the buffered body, so this also covers that the content survives the second +// attempt. func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { mw := mocks.NewMockWorkspaceClient(t) workspaceApi := mw.GetMockWorkspaceAPI() @@ -346,6 +368,8 @@ func TestWorkspaceFilesClientWriteCreatesParentDirectories(t *testing.T) { require.NoError(t, err) } +// Confirms a workspace get-status payload unmarshals into wsfsFileInfo and that +// the fs.FileInfo methods derived from it report the expected values. func TestWorkspaceFilesClient_wsfsUnmarshal(t *testing.T) { payload := ` { @@ -412,6 +436,8 @@ func statWithError(t *testing.T, statusCode int, errorCode string) error { return err } +// Confirms a 403 from get-status surfaces as an APIError rather than being +// remapped, so callers can distinguish it from a missing file. func TestWorkspaceFilesClientStatForbidden(t *testing.T) { err := statWithError(t, 403, "PERMISSION_DENIED") var apiErr *apierr.APIError @@ -419,6 +445,8 @@ func TestWorkspaceFilesClientStatForbidden(t *testing.T) { assert.Equal(t, 403, apiErr.StatusCode) } +// Confirms a 500 from get-status surfaces as an APIError and is not treated as +// a missing file. func TestWorkspaceFilesClientStatInternalError(t *testing.T) { err := statWithError(t, 500, "INTERNAL_ERROR") var apiErr *apierr.APIError @@ -426,6 +454,8 @@ func TestWorkspaceFilesClientStatInternalError(t *testing.T) { assert.Equal(t, 500, apiErr.StatusCode) } +// Confirms RESOURCE_DOES_NOT_EXIST maps to fs.ErrNotExist so the filer plugs +// into the io/fs error conventions callers check against. func TestWorkspaceFilesClientStatNotFound(t *testing.T) { err := statWithError(t, 404, "RESOURCE_DOES_NOT_EXIST") assert.ErrorIs(t, err, fs.ErrNotExist) diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index d3bb57662d9..fa7ff0baceb 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -10,7 +10,10 @@ import ( "golang.org/x/sync/errgroup" ) -// Maximum number of concurrent requests during sync. +// Maximum number of concurrent requests during sync. Chosen against the +// per-endpoint rate limits documented at +// https://docs.databricks.com/aws/en/resources/limits ("API rate limits" - +// Workspace API), which sync stays within for the endpoints it calls. const MaxRequestsInFlight = 20 // Delete the specified path. From 9a67c01f5a375050fbd99d946214bae58d2d15bf Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 5 Aug 2026 13:33:15 +0000 Subject: [PATCH 5/5] acceptance: migrate ai_runtime_task snapshot upload assertions The local_code_source test filtered recorded uploads by URL path, which no longer matches: /workspace/import carries the target filename in the multipart body. Filter on the body instead, the same way auto-migrate-empty-tfstate does. --del-field raw_body is dropped because the tarball is binary and the request recorder already summarizes it as a size placeholder. This test was added in #6110, after the upload migration branch was cut, so it was not covered by the earlier fixture updates. Co-authored-by: Isaac --- .../local_code_source/output.txt | 37 +++++++++++++------ .../ai_runtime_task/local_code_source/script | 15 +++++--- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt index 06cdf841c55..eded043b56b 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt +++ b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt @@ -21,19 +21,29 @@ src/train.py === both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create +>>> print_requests.py --sort //workspace/import //jobs/create { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 189 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 273 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } { @@ -114,12 +124,17 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ +>>> print_requests.py --sort //workspace/import { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", - "q": { - "overwrite": "true" + "path": "/api/2.0/workspace/import", + "body": { + "multipart_form": { + "content": "[binary content 276 bytes]", + "format": "AUTO", + "overwrite": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz" + } } } diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/script b/acceptance/bundle/ai_runtime_task/local_code_source/script index fe5e707ce01..6135bad0010 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/script +++ b/acceptance/bundle/ai_runtime_task/local_code_source/script @@ -11,10 +11,14 @@ title "each task's tarball holds only synced files (both under the repo root .ai trace list_code_snapshot.py title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n" -# --del-field raw_body drops the binary tarball upload payload (kept readable). Filters -# use a leading // so Git Bash on Windows does not path-convert them. --keep is not -# passed, so print_requests.py consumes out.requests.txt. -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create' +# Uploads go to POST /workspace/import, which carries the target filename in the +# multipart body rather than the URL, so .air_snapshots is matched with jq on the +# recorded body instead of by path filter. The tarball payload is binary and is +# already summarized as "[binary content N bytes]" by the request recorder. +# Filters use a leading // so Git Bash on Windows does not path-convert them. +# --keep is not passed, so print_requests.py consumes out.requests.txt. +trace print_requests.py --sort '//workspace/import' '//jobs/create' | + jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/|/jobs/create"))' title "re-planning unchanged code is a no-op (no changes)\n" trace $CLI bundle plan @@ -22,7 +26,8 @@ trace $CLI bundle plan title "editing a file changes the snapshot hash (content-addressed name changes)\n" update_file.py src/train.py 'print("training")' 'print("training v2")' trace $CLI bundle deploy -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' +trace print_requests.py --sort '//workspace/import' | + jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/"))' title "destroy removes the deployed bundle (including the synced snapshots)\n" trace $CLI bundle destroy --auto-approve