COR-181 - Make MailCore2 independent from AWS

.github/workflows/pull-request-check.yml

@@ -60,6 +60,30 @@ jobs:
     - name: Run connected android tests
       run: swift-android build --build-tests
 
+  Windows:
+    name: "mailcore2 - Windows prebuilt"
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+
+    # There is no Windows CI: the C/C++ for Windows is built by hand and published as a release
+    # asset named after a digest of the sources. This job does not build anything - it only asks
+    # whether the archive for these sources exists, so that a forgotten upload is caught in the
+    # pull request instead of in the spark-core build days later. The digest is a hash of git
+    # tree entries, identical on every platform, so a Linux runner answers it in seconds.
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2 # the pull request merge commit and both of its parents
+
+    - name: Check that a prebuilt is published for these sources
+      shell: pwsh
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: ./build-windows-5.10/Check-PrebuiltPublished.ps1
+
   Darwin:
     name: "mailcore2 - ${{ matrix.os }}"
     runs-on: macos-latest

AGENTS.md

@@ -0,0 +1,98 @@
+# Working in this repository
+
+This is Readdle's fork of MailCore 2. Most of it is the upstream C++ library; the parts we
+maintain are the C wrapper (`src/c`), the Swift bindings (`src/swift`) and the Windows build
+under `build-windows-5.10`.
+
+## Publishing the Windows prebuilt
+
+Spark's Windows build does not compile the mailcore2 C++ from source. It downloads an archive
+whose name is derived from the content of the C/C++ sources, so **sources that were never built
+simply have no archive** and the Spark build says so:
+
+```
+Prebuilt MailCore not found: mailcore2-windows-<digest>.zip
+Have you built and uploaded it yet?
+```
+
+There is no Windows CI. The `mailcore2 - Windows prebuilt` pull-request check asks the same
+question earlier, on every pull request, and goes red when the answer is no. Either way,
+someone has to build and publish from a Windows machine — that is what "build and upload the
+Windows changes" means.
+
+You can ask the same question yourself, from anywhere, without a Windows machine:
+
+```powershell
+pwsh ./build-windows-5.10/Check-PrebuiltPublished.ps1
+```
+
+### The whole procedure
+
+On a Windows build machine, in a checkout of this repository **at the revision that needs the
+prebuilt** (a branch head, a tag, anything committed):
+
+```powershell
+.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1
+```
+
+That is the entire job. It computes the digest, exits early if that archive already exists,
+fetches the dependency archive, clears the previous build output, builds through
+`Build-Mailcore2.ps1`, stamps the digest, packages, verifies the package, and adds it to the
+release.
+
+**Do not commit anything to mailcore2 afterwards.** The archive is named after the sources, so
+the revision that needs it will find it. Committing, opening the PR and tagging are the
+developer's job, not the agent's.
+
+### The release
+
+Archives live on the permanent `windows-prebuilt` release, set up by hand once.
+`Publish-Mailcore2Prebuilt.ps1` only adds `mailcore2-windows-<digest>.zip` to it. If it is
+somehow missing, say so rather than creating one.
+
+### Requirements
+
+The same machine that has always built mailcore2 for Windows: the toolchain the build expects
+(Swift for Windows, VS 2022 Build Tools, the Windows SDK) and the internal RD PowerShell
+modules. Nothing about the build changed.
+
+What did change: no AWS credential. `SPARK_PREBUILT_KEY` is gone, and the three binaries it
+used to fetch travel as one public asset on the release. Publishing additionally needs `gh`
+authenticated as a user with write access (`gh auth login`).
+
+The working tree must be committed. The digest describes `HEAD`, so uncommitted changes under
+`src` (excluding `src/swift`), `CMakeLists.txt` or `build-windows-5.10` make the script refuse
+to run.
+
+### When it fails
+
+- *Uncommitted changes under …* — commit them first, or test locally by passing
+  `-BuildMailcore2` to `Build-SwiftMailcore.ps1` instead of publishing.
+- *The windows-prebuilt release does not exist* — ask the developer; see "The release" above.
+- *Could not download the dependency archive* — `mailcore2-windows-deps-<n>.zip` carries
+  openssl, sasl and zlib and is attached to the release by hand. If it is not there, ask the
+  developer to attach it. The message names the path a local copy can be dropped at instead.
+- *GitHub CLI is not authenticated* — `gh auth login`. Do not work around this with a token
+  pasted into the shell.
+- Anything about ICU, libxml2 or the toolchain — that is the build machine's setup, unchanged by
+  this flow. Do not try to install or relocate it.
+
+### What not to do
+
+- Do not edit the archive by hand or upload one built from an uncommitted tree.
+- Do not delete a published `mailcore2-windows-<digest>.zip`: revisions that were built against
+  it keep downloading it by name. Re-uploading the *same* digest after a rebuild is the only
+  legitimate replacement, and `-Force` exists for exactly that.
+- Do not create the release or attach the dependency archive. Both are the developer's.
+- Do not add a version number anywhere. The digest replaced it; there is nothing to bump.
+
+### What the digest covers
+
+`src` (without `src/swift`), `CMakeLists.txt`, and `build-windows-5.10` — the scripts included,
+because they decide what gets compiled, which project files and redistributables get copied in,
+and what ends up in the package. So there is no way to change the contents of an archive without
+changing its name.
+
+The price is the other direction: editing any script here, even a message or the pull-request
+check, produces a new digest and asks for a rebuild. Expect that, and prefer to land script edits
+together with whatever else needs republishing.

README.md

@@ -34,10 +34,27 @@ Read [instructions for Linux](https://github.com/MailCore/mailcore2/blob/master/
 Windows builds of spark-core do **not** compile mailcore2 C++ from source.
 `Build-SparkCore.ps1` clones this repo at a pinned tag and runs
 `build-windows-5.10\Build-SwiftMailcore.ps1`, which compiles only the Swift
-bindings (`src/swift`) and downloads the prebuilt C++ libraries from S3
-(`Get-Mailcore2.ps1`, `mailcore2-all-<N>.zip`). **Any C++ change reaches
-Windows only through a new prebuilt archive** — merging a PR or moving a tag
-is not enough.
+bindings (`src/swift`) and downloads the prebuilt C++ libraries
+(`Get-Mailcore2.ps1`). **Any C++ change reaches Windows only through a new
+prebuilt archive** — merging a PR or moving a tag is not enough.
+
+The archive is a **release asset of this repository**, named after a digest of
+the sources it was built from:
+
+```text
+mailcore2-windows-<digest>.zip
+```
+
+It used to be `mailcore2-all-<N>.zip` in the `spark-prebuilt-binaries` S3
+bucket, behind `SPARK_PREBUILT_KEY`, with the version bumped by hand inside the
+shipped tag. Nothing about the build changed — only where the archive lives and
+how it is named. Two things follow:
+
+- **Nothing to bump.** The digest covers `src` (without `src/swift`),
+  `CMakeLists.txt` and `build-windows-5.10`, so a checkout already knows which
+  archive it needs. A change that touches none of those reuses the published one.
+- **A missing archive cannot pass silently.** The `mailcore2 - Windows prebuilt`
+  pull-request check says so before the spark-core build ever runs.
 
 ### Prerequisites ###
 
@@ -60,69 +77,86 @@ easiest path. On a bare machine you need:
   dispatch/BlocksRuntime).
 - ICU 69.1 at `C:\Library\icu-69.1\usr`, libxml2 2.11.5 at
   `C:\Library\libxml2-2.11.5\usr` (paths are hardcoded in the script).
-- `SPARK_PREBUILT_KEY` env var — download token for
-  `spark-prebuilt-binaries.s3.amazonaws.com` (zlib/sasl/openssl prebuilts).
 - ssh access to `git@github.com:readdle/{ctemplate,libetpan,tidy-html5}`.
 
+`SPARK_PREBUILT_KEY` is no longer needed: zlib/sasl/openssl come from a public
+release asset. Publishing additionally needs the GitHub CLI, authenticated as a
+user with write access — that one cannot be baked into the image:
+
+```powershell
+gh auth login
+```
+
 ### Build ###
 
 ```powershell
-$env:SPARK_PREBUILT_KEY = "<key>"
 powershell -ExecutionPolicy Bypass -File .\build-windows-5.10\Build-Mailcore2.ps1 -Install
 ```
 
 The script clones and builds ctemplate/libetpan/tidy, downloads the binary
-deps, then builds mailcore2/CMailCore with CMake + Ninja using `clang-cl`
-from the Swift toolchain. `-Install` lays the result out in `.build\install`
+deps (openssl/sasl/zlib, now from the release instead of S3), then builds
+mailcore2/CMailCore with CMake + Ninja using `clang-cl` from the Swift
+toolchain. `-Install` lays the result out in `.build\install`
 (`bin`, `include`, `lib`, `etc`).
 
 - After a **failed** run, delete `.build` before retrying — stale CMake
   caches keep the old configuration (wrong install prefix, wrong build type)
-  and produce confusing errors.
+  and produce confusing errors. `Publish-Mailcore2Prebuilt.ps1` does this for
+  you; a direct run does not.
 - Verify what was built: `type .build\install\etc\mailcore2-git-rev` must be
   the commit you intend to ship.
 
-### Package ###
+### Publish ###
 
-The zip must contain a single top-level folder named exactly `mailcore2-all`
-(that is the path `Get-Mailcore2.ps1` extracts):
+Packaging and uploading are one command, run in a checkout **at the revision
+that needs the prebuilt**:
 
 ```powershell
-cd .\.build
-Copy-Item -Recurse install mailcore2-all
-tar -a -cf mailcore2-all-<N+1>.zip mailcore2-all
+.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1
 ```
 
-Sanity check against the current archive: `tar -tf` both files and compare
-the top-level layout.
+It computes the digest, exits early if that archive is already published,
+fetches the dependency archive, clears the previous build output, builds through
+`Build-Mailcore2.ps1`, stamps `etc/mailcore2-source-digest`, packages
+`mailcore2-all/`, verifies the package, and adds it to the
+[`windows-prebuilt`](https://github.com/readdle/mailcore2/releases/tag/windows-prebuilt)
+release. `-Force` rebuilds and replaces an archive that is already published;
+`-SkipUpload` stops after verifying.
 
-### Upload ###
+Nothing needs to be committed afterwards — the archive is named after the
+sources, so the revision that needs it finds it.
 
-The bucket is `spark-prebuilt-binaries` in **eu-central-1**.
-`SPARK_PREBUILT_KEY` is a download-only token — uploads need real AWS
-credentials:
+The `windows-prebuilt` release is a permanent container for binaries, not a code
+release. It was created by hand, once, along with
+`mailcore2-windows-deps-1.zip`: openssl, sasl and zlib, the three binaries that
+used to come from S3, unchanged. No script creates the release or attaches that
+archive.
 
-```bash
-aws s3 cp mailcore2-all-<N+1>.zip s3://spark-prebuilt-binaries/mailcore2-all-<N+1>.zip --region eu-central-1
-```
+Never delete a published `mailcore2-windows-<digest>.zip` — revisions built
+against it keep downloading it by name.
 
-Verify the build can fetch it the same way the script does:
+### Pull request check ###
 
-```powershell
-Invoke-RestMethod -Method Head -Uri "https://spark-prebuilt-binaries.s3.amazonaws.com/mailcore2-all-<N+1>.zip" -UserAgent $env:SPARK_PREBUILT_KEY
-```
+`.github/workflows/pull-request-check.yml` runs
+[`build-windows-5.10/Check-PrebuiltPublished.ps1`](build-windows-5.10/Check-PrebuiltPublished.ps1)
+on every pull request. It builds nothing and needs no Windows: the digest is a
+hash of git tree entries, identical on every platform, so a Linux runner
+computes it in seconds and asks the release whether that archive is there.
+
+It checks the merge result, because that is what lands on the base branch and
+gets tagged. When the base has moved since you published, your branch head has
+an archive and the merge result does not — the check says so and tells you to
+rebase.
 
-Never overwrite an existing `mailcore2-all-<N>.zip` — older tags keep
-downloading it by name.
+Re-run it after publishing; nothing needs to be pushed for it to turn green. It
+is advisory until `mailcore2 - Windows prebuilt` is added to the branch
+protection rules for `spark2`.
 
 ### Switch builds to the new prebuilt ###
 
-1. In this repo: bump `$PrebuiltMailcoreVersion` in
-   `build-windows-5.10/Get-Mailcore2.ps1`, PR into `spark2`.
-2. Tag the merge with the next `2.1.x` tag. The bump **must be inside the
-   tag** — spark-core runs `Get-Mailcore2.ps1` from its mailcore checkout at
-   that tag, so a tag without the bump silently downloads the previous
-   archive.
+1. Publish the prebuilt for the revision you are shipping (above). There is no
+   version to bump and nothing to commit for it.
+2. Tag the merge with the next `2.1.x` tag.
 3. In `spark-core-mono`, update every mailcore pin to the new tag — the
    versions must match across platforms:
    - `spark-core/build-scripts/Windows-5.10/Build-SparkCore.ps1`

build-windows-5.10/Build-Mailcore2.ps1

@@ -8,6 +8,8 @@ Import-Module RDBuildCMake
 Import-Module RDBuildMSVC
 Import-Module RDDependency
 
+. "$PSScriptRoot\Prebuilt-Common.ps1"
+
 $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")"
 if (-Not $DependenciesPath) {
     $DependenciesPath = "$ProjectRoot\.build\Dependencies"
@@ -32,20 +34,14 @@ $LibEtPanDependencyDir = "LibEtPan"
 $LibEtPanDependencyPath = "$DependenciesPath\$LibEtPanDependencyDir"
 $TidyDependencyDir = "TidyHTML5"
 $TidyDependencyPath = "$DependenciesPath\$TidyDependencyDir"
-$ZlibDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/zlib.zip"
-$ZlibDependencyDir = "zlib"
-$ZlibDependencyPath = "$DependenciesPath\$ZlibDependencyDir\zlib-win32-1"
-$SaslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/sasl.zip"
-$SaslDependencyDir = "SASL"
-$SaslDependencyPath = "$DependenciesPath\$SaslDependencyDir\cyrus-sasl-win32"
-$OpenSslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/openssl.zip"
-$OpenSslDependencyDir = "OpenSSL"
-$OpenSslDependencyPath = "$DependenciesPath\$OpenSslDependencyDir\openssl-win32"
-
-$S3Key = $env:SPARK_PREBUILT_KEY
-if (!$S3Key) {
-    throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required"
-}
+# openssl, sasl and zlib used to come from the spark-prebuilt-binaries S3 bucket, one zip each,
+# behind SPARK_PREBUILT_KEY. They are the same binaries, now carried by a single public asset on
+# the prebuilt release, so no credential is involved. ICU and libxml2 never came from S3 and are
+# unchanged: they arrive with the toolchain, under C:\Library.
+$PrebuiltDependenciesRoot = "$DependenciesPath\mailcore2-windows-deps"
+$ZlibDependencyPath = "$PrebuiltDependenciesRoot\zlib"
+$SaslDependencyPath = "$PrebuiltDependenciesRoot\sasl"
+$OpenSslDependencyPath = "$PrebuiltDependenciesRoot\openssl"
 
 $Dependencies = @(
     @{ Name = "CTemplate"; GitUrl = "git@github.com:readdle/ctemplate.git"; GitBranch = "master"; Directory = $CTemplateDependencyDir; }
@@ -67,12 +63,16 @@ Push-Task -Name "mailcore2" -ScriptBlock {
         Write-TaskLog "Found Swift SDK: $SwiftSDKPath"
 
         Initialize-Dependencies -Path $Script:DependenciesPath -Dependencies $Script:Dependencies
-        Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key
-        Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key
-        Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key
-        Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force
-        Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force
-        Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force
+        Push-Task -Name "Fetch binary dependencies" -ScriptBlock {
+            $archive = Get-MailcoreDependenciesArchive -WorkPath $DependenciesPath
+            # A tree left by an older archive must not survive: files it no longer contains
+            # would otherwise still be picked up.
+            if (Test-Path -LiteralPath $PrebuiltDependenciesRoot) {
+                Remove-Item -LiteralPath $PrebuiltDependenciesRoot -Recurse -Force
+            }
+            Write-TaskLog "Extracting $archive"
+            Expand-Archive -Path $archive -DestinationPath $DependenciesPath -Force
+        }
 
         Push-Task -Name "Prepare Build Environment" -ScriptBlock {
             Test-Directory $IcuPath -SuccessMessage "Found ICU at $IcuPath" -FailMessage "ICU not found at $IcuPath"