Seeweb backend: pydantic v2 alignment and review fixes

.gitignore

@@ -22,7 +22,6 @@ venv/
 .env
 .envrc
 uv.lock
-/CI_TESTING.md
 
 /runner/cmd/shim/shim
 /runner/cmd/runner/runner

mkdocs/docs/concepts/backends.md

@@ -997,7 +997,7 @@ projects:
 ### Seeweb
 
 Log into the [Seeweb](https://www.seeweb.it/en) control panel, go to the
-`Calculation` section, open `API Token`, and click `New token` to generate one.
+`Compute` section, open `API Token`, and click `New token` to generate one.
 
 Then, go ahead and configure the backend:
 
@@ -1018,10 +1018,6 @@ projects:
 By default, `dstack` uses all Seeweb regions. To limit them, specify `regions`
 (for example, `it-fr2`).
 
-`dstack` checks [live plan capacity](https://docs.seeweb.it/en/hosting/cloudserver/rest-api/API-Endpoints/Plan/)
-before provisioning. A compatible NVIDIA driver image is selected from the images allowed by the
-chosen plan.
-
 ### OCI
 
 There are two ways to configure OCI: using client credentials or using the default credentials.

src/dstack/_internal/core/backends/seeweb/api_client.py

@@ -26,8 +26,8 @@ class SeewebPlanAvailability:
 class SeewebApiClient:
     """Minimal client for the Seeweb ECS REST API (https://api.seeweb.it/ecs/v2).
 
-    Uses ``requests`` directly instead of the ``ecsapi`` SDK: ``ecsapi`` requires pydantic v2,
-    while dstack is pinned to pydantic v1, so the two cannot share an environment.
+    Uses ``requests`` directly instead of the ``ecsapi`` SDK to avoid an extra dependency
+    for the handful of endpoints dstack needs.
     """
 
     def __init__(self, api_token: str):

src/dstack/_internal/core/backends/seeweb/compute.py

@@ -1,18 +1,22 @@
 import shlex
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import gpuhunt
 from gpuhunt.providers.seeweb import SeewebProvider
 
 from dstack._internal.core.backends.base.backend import Compute
 from dstack._internal.core.backends.base.compute import (
+    DEFAULT_PRIVATE_SUBNETS,
     ComputeWithCreateInstanceSupport,
     ComputeWithFilteredOffersCached,
     ComputeWithInstanceVolumesSupport,
     ComputeWithMultinodeSupport,
     ComputeWithPrivilegedSupport,
     generate_unique_instance_name,
-    get_shim_commands,
+    get_dstack_shim_binary_path,
+    get_setup_cloud_instance_commands,
+    get_shim_env,
+    get_shim_pre_start_commands,
 )
 from dstack._internal.core.backends.base.offers import get_catalog_offers
 from dstack._internal.core.backends.seeweb.api_client import (
@@ -22,7 +26,7 @@
 from dstack._internal.core.backends.seeweb.models import SeewebConfig
 from dstack._internal.core.errors import BackendError, NoCapacityError, ProvisioningError
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import CoreModel
+from dstack._internal.core.models.common import CoreModel, validate_json_extra_ignore
 from dstack._internal.core.models.instances import (
     InstanceAvailability,
     InstanceConfiguration,
@@ -37,10 +41,11 @@
 # Seeweb auto-generates the server name; notes are limited, so keep the label short.
 MAX_INSTANCE_NAME_LEN = 60
 
-# Seeweb statuses that mean provisioning failed (matched case-insensitively).
-FAILED_SERVER_STATUSES = {"failed", "error", "deleted", "deleting"}
-FAILED_ACTION_STATUSES = {"failed", "error", "cancelled", "canceled"}
-COMPLETED_ACTION_STATUSES = {"completed", "complete", "success", "succeeded"}
+# Statuses from Seeweb's `ecsapi` SDK (`ServerStatusEnum`/`ActionStatusEnum`),
+# matched case-insensitively; the API reports them capitalized (e.g. "Fail").
+FAILED_SERVER_STATUSES = {"fail", "deleted", "deleting"}
+FAILED_ACTION_STATUSES = {"failed"}
+COMPLETED_ACTION_STATUSES = {"completed"}
 
 GPU_IMAGE_PREFERENCE = (
     "ubuntu-2204-uefi-nvidia-driver",
@@ -70,7 +75,7 @@ def _make_catalog(self) -> gpuhunt.Catalog:
         return catalog
 
     def get_offers_by_requirements(
-        self, requirements: Requirements, full_offers: bool
+        self, requirements: Requirements, full_offers: bool, unallocated_resources: bool
     ) -> List[InstanceOfferWithAvailability]:
         offers = get_catalog_offers(
             backend=BackendType.SEEWEB,
@@ -111,13 +116,14 @@ def create_instance(
             is_gpu=is_gpu,
         )
 
+        arch = instance_offer.instance.resources.cpu_arch
         commands = _setup_commands(authorized_keys=public_keys, is_gpu=is_gpu)
-        shim_commands = get_shim_commands(arch=instance_offer.instance.resources.cpu_arch)
-        # The Seeweb GPU image reboots after its first-boot package upgrade. Do not start
-        # the shim before that reboot, otherwise dstack can submit a GPU job while the
-        # userspace NVIDIA libraries no longer match the still-loaded kernel module.
-        commands += shim_commands[:-1]
-        commands += _persist_shim_commands()
+        commands += get_setup_cloud_instance_commands(
+            skip_firewall_setup=False,
+            firewall_allow_from_subnets=DEFAULT_PRIVATE_SUBNETS,
+        )
+        commands += get_shim_pre_start_commands(arch=arch)
+        commands += _persist_shim_commands(get_shim_env(arch=arch))
         user_customize = "#!/bin/bash\nset -euo pipefail\n" + "\n".join(commands)
 
         body = {
@@ -146,7 +152,7 @@ def create_instance(
             ssh_port=22,
             ssh_proxy=None,
             dockerized=True,
-            backend_data=SeewebInstanceBackendData(action_id=action_id).json(),
+            backend_data=SeewebInstanceBackendData(action_id=action_id).model_dump_json(),
         )
 
     def update_provisioning_data(
@@ -253,8 +259,10 @@ def _install_nvidia_container_toolkit_commands() -> List[str]:
     ]
 
 
-def _persist_shim_commands() -> List[str]:
-    """Make the shim survive the reboot performed by Seeweb's GPU image updates."""
+def _persist_shim_commands(shim_env: Dict[str, str]) -> List[str]:
+    """Run the shim as a systemd service so it starts after Seeweb's first-boot reboot."""
+    env_lines = [f"{key}={value}" for key, value in shim_env.items()]
+    write_env = "printf '%s\\n' " + " ".join(map(shlex.quote, env_lines))
     unit_lines = [
         "[Unit]",
         "Description=dstack shim",
@@ -263,7 +271,7 @@ def _persist_shim_commands() -> List[str]:
         "",
         "[Service]",
         "EnvironmentFile=/etc/dstack-shim.env",
-        "ExecStart=/usr/local/bin/dstack-shim",
+        f"ExecStart={get_dstack_shim_binary_path()}",
         "Restart=always",
         "RestartSec=3",
         "",
@@ -272,12 +280,13 @@ def _persist_shim_commands() -> List[str]:
     ]
     write_unit = "printf '%s\\n' " + " ".join(map(shlex.quote, unit_lines))
     return [
-        "env | grep '^DSTACK_' > /etc/dstack-shim.env",
+        f"{write_env} > /etc/dstack-shim.env",
         "chmod 0600 /etc/dstack-shim.env",
         f"{write_unit} > /etc/systemd/system/dstack-shim.service",
         "systemctl daemon-reload",
-        # Do not start the service on the initial boot. Seeweb's GPU image performs an
-        # automatic reboot after cloud-init, and the service must become reachable only then.
+        # Only enable, never start: Seeweb's cloud-config unconditionally reboots the
+        # server at the end of first-boot cloud-init (`power_state`), and the service
+        # must become reachable only on that next boot.
         "systemctl enable dstack-shim.service",
     ]
 
@@ -289,4 +298,4 @@ class SeewebInstanceBackendData(CoreModel):
     def load(cls, raw: Optional[str]) -> "SeewebInstanceBackendData":
         if raw is None:
             return cls()
-        return cls.__response__.parse_raw(raw)
+        return validate_json_extra_ignore(cls, raw)

src/dstack/_internal/core/backends/seeweb/configurator.py

@@ -15,6 +15,7 @@
     SeewebStoredConfig,
 )
 from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.common import validate_extra_ignore, validate_json_extra_ignore
 
 
 class SeewebConfigurator(
@@ -35,25 +36,28 @@ def create_backend(
     ) -> BackendRecord:
         return BackendRecord(
             config=SeewebStoredConfig(
-                **SeewebBackendConfig.__response__.parse_obj(config).dict()
-            ).json(),
-            auth=SeewebCreds.parse_obj(config.creds).json(),
+                **validate_extra_ignore(SeewebBackendConfig, config).model_dump()
+            ).model_dump_json(),
+            auth=SeewebCreds.model_validate(config.creds).model_dump_json(),
         )
 
     def get_backend_config_with_creds(self, record: BackendRecord) -> SeewebBackendConfigWithCreds:
         config = self._get_config(record)
-        return SeewebBackendConfigWithCreds.__response__.parse_obj(config)
+        return validate_extra_ignore(SeewebBackendConfigWithCreds, config)
 
     def get_backend_config_without_creds(self, record: BackendRecord) -> SeewebBackendConfig:
         config = self._get_config(record)
-        return SeewebBackendConfig.__response__.parse_obj(config)
+        return validate_extra_ignore(SeewebBackendConfig, config)
 
     def get_backend(self, record: BackendRecord) -> SeewebBackend:
         config = self._get_config(record)
         return SeewebBackend(config=config)
 
     def _get_config(self, record: BackendRecord) -> SeewebConfig:
-        return SeewebConfig.__response__(
-            **json.loads(record.config),
-            creds=SeewebCreds.parse_raw(record.auth),
+        return validate_extra_ignore(
+            SeewebConfig,
+            {
+                **json.loads(record.config),
+                "creds": validate_json_extra_ignore(SeewebCreds, record.auth),
+            },
         )

src/tests/_internal/core/backends/seeweb/test_compute.py

@@ -2,7 +2,7 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-from gpuhunt import RawCatalogItem
+from gpuhunt import CatalogItem
 
 from dstack._internal.core.backends.base.compute import (
     ComputeWithInstanceVolumesSupport,
@@ -41,8 +41,9 @@ def _compute(regions=None) -> SeewebCompute:
     return compute
 
 
-def _raw(name: str, gpu: str, location: str = "it-fr2") -> RawCatalogItem:
-    return RawCatalogItem(
+def _raw(name: str, gpu: str, location: str = "it-fr2") -> CatalogItem:
+    return CatalogItem(
+        provider="seeweb",
         instance_name=name,
         location=location,
         price=0.38,
@@ -98,7 +99,7 @@ def _provisioning_data(action_id: int | None = 35) -> JobProvisioningData:
         ssh_port=22,
         dockerized=True,
         ssh_proxy=None,
-        backend_data=SeewebInstanceBackendData(action_id=action_id).json(),
+        backend_data=SeewebInstanceBackendData(action_id=action_id).model_dump_json(),
     )
 
 
@@ -122,7 +123,7 @@ def test_get_offers_marks_only_creatable_as_available():
         ),
     ):
         offers = SeewebCompute(_config()).get_offers_by_requirements(
-            requirements=None, full_offers=False
+            requirements=None, full_offers=False, unallocated_resources=False
         )
 
     by_name = {offer.instance.name: offer for offer in offers}
@@ -148,7 +149,7 @@ def test_get_offers_filters_configured_regions():
         ),
     ):
         offers = SeewebCompute(_config(["it-mi2"])).get_offers_by_requirements(
-            requirements=None, full_offers=False
+            requirements=None, full_offers=False, unallocated_resources=False
         )
 
     assert [offer.region for offer in offers] == ["it-mi2"]
@@ -217,8 +218,12 @@ def test_create_instance_uses_allowed_image_and_quotes_ssh_key():
             return_value="dstack-seeweb-test",
         ),
         patch(
-            "dstack._internal.core.backends.seeweb.compute.get_shim_commands",
-            return_value=["prepare-shim", "start-shim"],
+            "dstack._internal.core.backends.seeweb.compute.get_shim_pre_start_commands",
+            return_value=["prepare-shim"],
+        ),
+        patch(
+            "dstack._internal.core.backends.seeweb.compute.get_shim_env",
+            return_value={"DSTACK_SHIM_HTTP_PORT": "10998"},
         ),
     ):
         provisioning_data = compute.create_instance(
@@ -233,7 +238,10 @@ def test_create_instance_uses_allowed_image_and_quotes_ssh_key():
     assert f"printf '%s\\n' {shlex.quote(public_key)}" in body["user_customize"]
     assert 'echo "ssh-' not in body["user_customize"]
     assert "prepare-shim" in body["user_customize"]
-    assert "start-shim" not in body["user_customize"]
+    # The shim must not be started on the first boot; systemd starts it after the reboot.
+    assert "nohup" not in body["user_customize"]
+    assert "DSTACK_SHIM_HTTP_PORT=10998" in body["user_customize"]
+    assert "> /etc/dstack-shim.env" in body["user_customize"]
     assert "EnvironmentFile=/etc/dstack-shim.env" in body["user_customize"]
     assert body["user_customize"].endswith("systemctl enable dstack-shim.service")
     assert provisioning_data.instance_id == "ec-test"
@@ -280,10 +288,10 @@ def test_update_provisioning_data_raises_for_failed_server():
     compute = _compute()
     compute.api_client.get_server.return_value = {
         "name": "ec-test",
-        "status": "Error",
+        "status": "Fail",
     }
 
-    with pytest.raises(ProvisioningError, match="entered status 'error'"):
+    with pytest.raises(ProvisioningError, match="entered status 'fail'"):
         compute.update_provisioning_data(_provisioning_data(action_id=None), "public", "private")