diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index d2e30d258..e251fbd5c 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -60,6 +60,30 @@ jobs: - name: Run connected android tests run: swift-android build --build-tests + Windows: + name: "mailcore2 - Windows prebuilt" + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + # There is no Windows CI: the C/C++ for Windows is built by hand and published as a release + # asset named after a digest of the sources. This job does not build anything - it only asks + # whether the archive for these sources exists, so that a forgotten upload is caught in the + # pull request instead of in the spark-core build days later. The digest is a hash of git + # tree entries, identical on every platform, so a Linux runner answers it in seconds. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 # the pull request merge commit and both of its parents + + - name: Check that a prebuilt is published for these sources + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: ./build-windows-5.10/Check-PrebuiltPublished.ps1 + Darwin: name: "mailcore2 - ${{ matrix.os }}" runs-on: macos-latest diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..97e1ffc2c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,98 @@ +# Working in this repository + +This is Readdle's fork of MailCore 2. Most of it is the upstream C++ library; the parts we +maintain are the C wrapper (`src/c`), the Swift bindings (`src/swift`) and the Windows build +under `build-windows-5.10`. + +## Publishing the Windows prebuilt + +Spark's Windows build does not compile the mailcore2 C++ from source. It downloads an archive +whose name is derived from the content of the C/C++ sources, so **sources that were never built +simply have no archive** and the Spark build says so: + +``` +Prebuilt MailCore not found: mailcore2-windows-.zip +Have you built and uploaded it yet? +``` + +There is no Windows CI. The `mailcore2 - Windows prebuilt` pull-request check asks the same +question earlier, on every pull request, and goes red when the answer is no. Either way, +someone has to build and publish from a Windows machine — that is what "build and upload the +Windows changes" means. + +You can ask the same question yourself, from anywhere, without a Windows machine: + +```powershell +pwsh ./build-windows-5.10/Check-PrebuiltPublished.ps1 +``` + +### The whole procedure + +On a Windows build machine, in a checkout of this repository **at the revision that needs the +prebuilt** (a branch head, a tag, anything committed): + +```powershell +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 +``` + +That is the entire job. It computes the digest, exits early if that archive already exists, +fetches the dependency archive, clears the previous build output, builds through +`Build-Mailcore2.ps1`, stamps the digest, packages, verifies the package, and adds it to the +release. + +**Do not commit anything to mailcore2 afterwards.** The archive is named after the sources, so +the revision that needs it will find it. Committing, opening the PR and tagging are the +developer's job, not the agent's. + +### The release + +Archives live on the permanent `windows-prebuilt` release, set up by hand once. +`Publish-Mailcore2Prebuilt.ps1` only adds `mailcore2-windows-.zip` to it. If it is +somehow missing, say so rather than creating one. + +### Requirements + +The same machine that has always built mailcore2 for Windows: the toolchain the build expects +(Swift for Windows, VS 2022 Build Tools, the Windows SDK) and the internal RD PowerShell +modules. Nothing about the build changed. + +What did change: no AWS credential. `SPARK_PREBUILT_KEY` is gone, and the three binaries it +used to fetch travel as one public asset on the release. Publishing additionally needs `gh` +authenticated as a user with write access (`gh auth login`). + +The working tree must be committed. The digest describes `HEAD`, so uncommitted changes under +`src` (excluding `src/swift`), `CMakeLists.txt` or `build-windows-5.10` make the script refuse +to run. + +### When it fails + +- *Uncommitted changes under …* — commit them first, or test locally by passing + `-BuildMailcore2` to `Build-SwiftMailcore.ps1` instead of publishing. +- *The windows-prebuilt release does not exist* — ask the developer; see "The release" above. +- *Could not download the dependency archive* — `mailcore2-windows-deps-.zip` carries + openssl, sasl and zlib and is attached to the release by hand. If it is not there, ask the + developer to attach it. The message names the path a local copy can be dropped at instead. +- *GitHub CLI is not authenticated* — `gh auth login`. Do not work around this with a token + pasted into the shell. +- Anything about ICU, libxml2 or the toolchain — that is the build machine's setup, unchanged by + this flow. Do not try to install or relocate it. + +### What not to do + +- Do not edit the archive by hand or upload one built from an uncommitted tree. +- Do not delete a published `mailcore2-windows-.zip`: revisions that were built against + it keep downloading it by name. Re-uploading the *same* digest after a rebuild is the only + legitimate replacement, and `-Force` exists for exactly that. +- Do not create the release or attach the dependency archive. Both are the developer's. +- Do not add a version number anywhere. The digest replaced it; there is nothing to bump. + +### What the digest covers + +`src` (without `src/swift`), `CMakeLists.txt`, and `build-windows-5.10` — the scripts included, +because they decide what gets compiled, which project files and redistributables get copied in, +and what ends up in the package. So there is no way to change the contents of an archive without +changing its name. + +The price is the other direction: editing any script here, even a message or the pull-request +check, produces a new digest and asks for a rebuild. Expect that, and prefer to land script edits +together with whatever else needs republishing. diff --git a/README.md b/README.md index 57520e56e..042e1ed20 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,27 @@ Read [instructions for Linux](https://github.com/MailCore/mailcore2/blob/master/ Windows builds of spark-core do **not** compile mailcore2 C++ from source. `Build-SparkCore.ps1` clones this repo at a pinned tag and runs `build-windows-5.10\Build-SwiftMailcore.ps1`, which compiles only the Swift -bindings (`src/swift`) and downloads the prebuilt C++ libraries from S3 -(`Get-Mailcore2.ps1`, `mailcore2-all-.zip`). **Any C++ change reaches -Windows only through a new prebuilt archive** — merging a PR or moving a tag -is not enough. +bindings (`src/swift`) and downloads the prebuilt C++ libraries +(`Get-Mailcore2.ps1`). **Any C++ change reaches Windows only through a new +prebuilt archive** — merging a PR or moving a tag is not enough. + +The archive is a **release asset of this repository**, named after a digest of +the sources it was built from: + +```text +mailcore2-windows-.zip +``` + +It used to be `mailcore2-all-.zip` in the `spark-prebuilt-binaries` S3 +bucket, behind `SPARK_PREBUILT_KEY`, with the version bumped by hand inside the +shipped tag. Nothing about the build changed — only where the archive lives and +how it is named. Two things follow: + +- **Nothing to bump.** The digest covers `src` (without `src/swift`), + `CMakeLists.txt` and `build-windows-5.10`, so a checkout already knows which + archive it needs. A change that touches none of those reuses the published one. +- **A missing archive cannot pass silently.** The `mailcore2 - Windows prebuilt` + pull-request check says so before the spark-core build ever runs. ### Prerequisites ### @@ -60,69 +77,86 @@ easiest path. On a bare machine you need: dispatch/BlocksRuntime). - ICU 69.1 at `C:\Library\icu-69.1\usr`, libxml2 2.11.5 at `C:\Library\libxml2-2.11.5\usr` (paths are hardcoded in the script). -- `SPARK_PREBUILT_KEY` env var — download token for - `spark-prebuilt-binaries.s3.amazonaws.com` (zlib/sasl/openssl prebuilts). - ssh access to `git@github.com:readdle/{ctemplate,libetpan,tidy-html5}`. +`SPARK_PREBUILT_KEY` is no longer needed: zlib/sasl/openssl come from a public +release asset. Publishing additionally needs the GitHub CLI, authenticated as a +user with write access — that one cannot be baked into the image: + +```powershell +gh auth login +``` + ### Build ### ```powershell -$env:SPARK_PREBUILT_KEY = "" powershell -ExecutionPolicy Bypass -File .\build-windows-5.10\Build-Mailcore2.ps1 -Install ``` The script clones and builds ctemplate/libetpan/tidy, downloads the binary -deps, then builds mailcore2/CMailCore with CMake + Ninja using `clang-cl` -from the Swift toolchain. `-Install` lays the result out in `.build\install` +deps (openssl/sasl/zlib, now from the release instead of S3), then builds +mailcore2/CMailCore with CMake + Ninja using `clang-cl` from the Swift +toolchain. `-Install` lays the result out in `.build\install` (`bin`, `include`, `lib`, `etc`). - After a **failed** run, delete `.build` before retrying — stale CMake caches keep the old configuration (wrong install prefix, wrong build type) - and produce confusing errors. + and produce confusing errors. `Publish-Mailcore2Prebuilt.ps1` does this for + you; a direct run does not. - Verify what was built: `type .build\install\etc\mailcore2-git-rev` must be the commit you intend to ship. -### Package ### +### Publish ### -The zip must contain a single top-level folder named exactly `mailcore2-all` -(that is the path `Get-Mailcore2.ps1` extracts): +Packaging and uploading are one command, run in a checkout **at the revision +that needs the prebuilt**: ```powershell -cd .\.build -Copy-Item -Recurse install mailcore2-all -tar -a -cf mailcore2-all-.zip mailcore2-all +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 ``` -Sanity check against the current archive: `tar -tf` both files and compare -the top-level layout. +It computes the digest, exits early if that archive is already published, +fetches the dependency archive, clears the previous build output, builds through +`Build-Mailcore2.ps1`, stamps `etc/mailcore2-source-digest`, packages +`mailcore2-all/`, verifies the package, and adds it to the +[`windows-prebuilt`](https://github.com/readdle/mailcore2/releases/tag/windows-prebuilt) +release. `-Force` rebuilds and replaces an archive that is already published; +`-SkipUpload` stops after verifying. -### Upload ### +Nothing needs to be committed afterwards — the archive is named after the +sources, so the revision that needs it finds it. -The bucket is `spark-prebuilt-binaries` in **eu-central-1**. -`SPARK_PREBUILT_KEY` is a download-only token — uploads need real AWS -credentials: +The `windows-prebuilt` release is a permanent container for binaries, not a code +release. It was created by hand, once, along with +`mailcore2-windows-deps-1.zip`: openssl, sasl and zlib, the three binaries that +used to come from S3, unchanged. No script creates the release or attaches that +archive. -```bash -aws s3 cp mailcore2-all-.zip s3://spark-prebuilt-binaries/mailcore2-all-.zip --region eu-central-1 -``` +Never delete a published `mailcore2-windows-.zip` — revisions built +against it keep downloading it by name. -Verify the build can fetch it the same way the script does: +### Pull request check ### -```powershell -Invoke-RestMethod -Method Head -Uri "https://spark-prebuilt-binaries.s3.amazonaws.com/mailcore2-all-.zip" -UserAgent $env:SPARK_PREBUILT_KEY -``` +`.github/workflows/pull-request-check.yml` runs +[`build-windows-5.10/Check-PrebuiltPublished.ps1`](build-windows-5.10/Check-PrebuiltPublished.ps1) +on every pull request. It builds nothing and needs no Windows: the digest is a +hash of git tree entries, identical on every platform, so a Linux runner +computes it in seconds and asks the release whether that archive is there. + +It checks the merge result, because that is what lands on the base branch and +gets tagged. When the base has moved since you published, your branch head has +an archive and the merge result does not — the check says so and tells you to +rebase. -Never overwrite an existing `mailcore2-all-.zip` — older tags keep -downloading it by name. +Re-run it after publishing; nothing needs to be pushed for it to turn green. It +is advisory until `mailcore2 - Windows prebuilt` is added to the branch +protection rules for `spark2`. ### Switch builds to the new prebuilt ### -1. In this repo: bump `$PrebuiltMailcoreVersion` in - `build-windows-5.10/Get-Mailcore2.ps1`, PR into `spark2`. -2. Tag the merge with the next `2.1.x` tag. The bump **must be inside the - tag** — spark-core runs `Get-Mailcore2.ps1` from its mailcore checkout at - that tag, so a tag without the bump silently downloads the previous - archive. +1. Publish the prebuilt for the revision you are shipping (above). There is no + version to bump and nothing to commit for it. +2. Tag the merge with the next `2.1.x` tag. 3. In `spark-core-mono`, update every mailcore pin to the new tag — the versions must match across platforms: - `spark-core/build-scripts/Windows-5.10/Build-SparkCore.ps1` diff --git a/build-windows-5.10/Build-Mailcore2.ps1 b/build-windows-5.10/Build-Mailcore2.ps1 index 4bcbf2f4a..5111aefa9 100644 --- a/build-windows-5.10/Build-Mailcore2.ps1 +++ b/build-windows-5.10/Build-Mailcore2.ps1 @@ -8,6 +8,8 @@ Import-Module RDBuildCMake Import-Module RDBuildMSVC Import-Module RDDependency +. "$PSScriptRoot\Prebuilt-Common.ps1" + $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $DependenciesPath) { $DependenciesPath = "$ProjectRoot\.build\Dependencies" @@ -32,20 +34,14 @@ $LibEtPanDependencyDir = "LibEtPan" $LibEtPanDependencyPath = "$DependenciesPath\$LibEtPanDependencyDir" $TidyDependencyDir = "TidyHTML5" $TidyDependencyPath = "$DependenciesPath\$TidyDependencyDir" -$ZlibDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/zlib.zip" -$ZlibDependencyDir = "zlib" -$ZlibDependencyPath = "$DependenciesPath\$ZlibDependencyDir\zlib-win32-1" -$SaslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/sasl.zip" -$SaslDependencyDir = "SASL" -$SaslDependencyPath = "$DependenciesPath\$SaslDependencyDir\cyrus-sasl-win32" -$OpenSslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/openssl.zip" -$OpenSslDependencyDir = "OpenSSL" -$OpenSslDependencyPath = "$DependenciesPath\$OpenSslDependencyDir\openssl-win32" - -$S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { - throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" -} +# openssl, sasl and zlib used to come from the spark-prebuilt-binaries S3 bucket, one zip each, +# behind SPARK_PREBUILT_KEY. They are the same binaries, now carried by a single public asset on +# the prebuilt release, so no credential is involved. ICU and libxml2 never came from S3 and are +# unchanged: they arrive with the toolchain, under C:\Library. +$PrebuiltDependenciesRoot = "$DependenciesPath\mailcore2-windows-deps" +$ZlibDependencyPath = "$PrebuiltDependenciesRoot\zlib" +$SaslDependencyPath = "$PrebuiltDependenciesRoot\sasl" +$OpenSslDependencyPath = "$PrebuiltDependenciesRoot\openssl" $Dependencies = @( @{ Name = "CTemplate"; GitUrl = "git@github.com:readdle/ctemplate.git"; GitBranch = "master"; Directory = $CTemplateDependencyDir; } @@ -67,12 +63,16 @@ Push-Task -Name "mailcore2" -ScriptBlock { Write-TaskLog "Found Swift SDK: $SwiftSDKPath" Initialize-Dependencies -Path $Script:DependenciesPath -Dependencies $Script:Dependencies - Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key - Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force + Push-Task -Name "Fetch binary dependencies" -ScriptBlock { + $archive = Get-MailcoreDependenciesArchive -WorkPath $DependenciesPath + # A tree left by an older archive must not survive: files it no longer contains + # would otherwise still be picked up. + if (Test-Path -LiteralPath $PrebuiltDependenciesRoot) { + Remove-Item -LiteralPath $PrebuiltDependenciesRoot -Recurse -Force + } + Write-TaskLog "Extracting $archive" + Expand-Archive -Path $archive -DestinationPath $DependenciesPath -Force + } Push-Task -Name "Prepare Build Environment" -ScriptBlock { Test-Directory $IcuPath -SuccessMessage "Found ICU at $IcuPath" -FailMessage "ICU not found at $IcuPath" diff --git a/build-windows-5.10/Check-PrebuiltPublished.ps1 b/build-windows-5.10/Check-PrebuiltPublished.ps1 new file mode 100644 index 000000000..74dea635c --- /dev/null +++ b/build-windows-5.10/Check-PrebuiltPublished.ps1 @@ -0,0 +1,74 @@ +# Answers one question: do the C/C++ sources in this checkout have a published Windows prebuilt? +# +# This is what the Windows pull-request check runs. It builds nothing and needs no Windows - the +# digest is a hash of git tree entries, identical on every platform - so a Linux runner answers +# in seconds. + +Param( + # Defaults to whatever is checked out, which on a pull request is the merge result: that is + # what lands on the base branch and gets tagged. + [string]$Ref = "HEAD" +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" +$Readme = "https://github.com/$Script:MailcorePrebuiltRepo/blob/spark2/README.md#windows-prebuilt" + +function Get-ArchiveFor { + param([string]$AtRef) + return Get-MailcorePrebuiltArchiveName -Digest (Get-MailcoreSourceDigest -RepoRoot $ProjectRoot -Ref $AtRef) +} + +if (-not (Test-MailcorePrebuiltRelease)) { + Write-Host "" + Write-Host $Script:MailcoreReleaseSetupHelp -ForegroundColor Red + Write-Host "" + Write-Host "::error title=Windows prebuilt release missing::The $Script:MailcorePrebuiltReleaseTag release does not exist. It is created once, by hand." + exit 1 +} + +$Archive = Get-ArchiveFor -AtRef $Ref +$Published = Get-MailcoreReleaseAssetNames + +Write-Host "" +Write-Host " commit : $((& git -C $ProjectRoot rev-parse $Ref).Trim())" -ForegroundColor Cyan +Write-Host " archive : $Archive" -ForegroundColor Cyan +Write-Host "" + +if ($Published -contains $Archive) { + Write-Host "Published. These sources have a Windows prebuilt." -ForegroundColor Green + Write-Host "" + exit 0 +} + +Write-Host "There is no Windows prebuilt for these sources: $Archive" -ForegroundColor Red +Write-Host "Have you built and uploaded it yet?" -ForegroundColor Red +Write-Host "" + +# On a merge commit the second parent is the pull request head - what a developer would have +# published from. When the two differ the base has moved, and saying so beats an unexplained red. +$parentLine = (& git -C $ProjectRoot rev-list --parents -n 1 $Ref 2>$null) +$parents = @("$parentLine".Trim() -split "\s+" | Where-Object { $_ }) +$headArchive = if ($parents.Count -ge 3) { Get-ArchiveFor -AtRef $parents[2] } else { $Archive } + +if ($headArchive -ne $Archive -and ($Published -contains $headArchive)) { + Write-Host "Your branch head has one ($headArchive), so the base branch moved after you" -ForegroundColor Yellow + Write-Host "published. Rebase onto the base branch and publish again from the rebased head." -ForegroundColor Yellow +} +else { + Write-Host "If not: on the Windows build machine, in a checkout of this branch," -ForegroundColor Yellow + Write-Host "" + Write-Host " .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1" + Write-Host "" + Write-Host "and re-run this check. Publishing commits nothing - the archive is named after the" -ForegroundColor Yellow + Write-Host "sources, so this revision will find it." -ForegroundColor Yellow +} + +Write-Host "" +Write-Host "How this works: $Readme" -ForegroundColor DarkGray +Write-Host "" +Write-Host "::error title=Windows prebuilt missing::No $Archive published - have you built and uploaded it yet? See $Readme" +exit 1 diff --git a/build-windows-5.10/Get-Mailcore2.ps1 b/build-windows-5.10/Get-Mailcore2.ps1 index 8b5429d8d..44bc23113 100644 --- a/build-windows-5.10/Get-Mailcore2.ps1 +++ b/build-windows-5.10/Get-Mailcore2.ps1 @@ -2,20 +2,20 @@ Param( [string]$InstallPath ) +. "$PSScriptRoot\Prebuilt-Common.ps1" + $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $InstallPath) { $InstallPath = "$ProjectRoot\.build\install" } -$PrebuiltMailcoreVersion = 5 -$PrebuiltMailcoreArchive = "mailcore2-all-$PrebuiltMailcoreVersion.zip" -$PrebuiltMailcoreUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/$PrebuiltMailcoreArchive" - -$S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { - throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" -} +# The archive is named after the content of the C/C++ sources, so this checkout already knows +# which one it needs - there is no version to keep in step. It is a public release asset, so +# downloading it takes no credential. +$PrebuiltMailcoreDigest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot +$PrebuiltMailcoreArchive = Get-MailcorePrebuiltArchiveName -Digest $PrebuiltMailcoreDigest +$PrebuiltMailcoreUrl = Get-MailcorePrebuiltUrl -ArchiveName $PrebuiltMailcoreArchive Push-Task -Name "mailcore2" -ScriptBlock { Push-Task -Name "Initialize" -ScriptBlock { @@ -31,12 +31,40 @@ Push-Task -Name "mailcore2" -ScriptBlock { Write-TaskLog "Downloading $PrebuiltMailcoreUrl to $TempFile" - Invoke-RestMethod -Uri $PrebuiltMailcoreUrl -OutFile $TempFile -UserAgent $S3Key + try { + Invoke-RestMethod -Uri $PrebuiltMailcoreUrl -OutFile $TempFile + } + catch { + # Only a 404 means "nobody published this yet". Anything else - a proxy, a dropped + # connection, a 5xx - must not be reported as a missing archive. + $Status = Get-MailcoreHttpStatus -ErrorRecord $_ + if ($Status -ne 404) { + throw "Could not download $PrebuiltMailcoreUrl$(if ($Status) { " (HTTP $Status)" }). $($_.Exception.Message)" + } + throw @" +Prebuilt MailCore not found: $PrebuiltMailcoreArchive +Have you built and uploaded it yet? + +These C/C++ sources (digest $PrebuiltMailcoreDigest) have no published Windows prebuilt. +On a Windows build machine, in a checkout at this exact revision, run: + + .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 + +Nothing needs to be committed afterwards - the archive is named after the sources, so this +revision will find it. To build the C++ from source instead, pass -BuildMailcore2 to +Build-SwiftMailcore.ps1. +"@ + } Remove-Item $TempDir -Force -Recurse -ErrorAction Ignore New-Item -ItemType Directory $TempDir Write-TaskLog "Extracting $TempFile to $TempDir" tar -C "$TempDir" -xf "$TempFile" + $ArchiveDigest = Get-MailcoreArchiveDigest -UnpackedPath "$TempDir\mailcore2-all" + if ($ArchiveDigest -ne $PrebuiltMailcoreDigest) { + throw "Prebuilt $PrebuiltMailcoreArchive was built from sources with digest '$ArchiveDigest', expected '$PrebuiltMailcoreDigest'. The published asset does not match its name." + } + New-Item -Path $InstallPath -ItemType Directory -ErrorAction Ignore Get-ChildItem -Path "$TempDir\mailcore2-all" | Copy-Item -Destination $InstallPath -Recurse -Container -PassThru -Force | Write-Host diff --git a/build-windows-5.10/Prebuilt-Common.ps1 b/build-windows-5.10/Prebuilt-Common.ps1 new file mode 100644 index 000000000..e5e5a15c7 --- /dev/null +++ b/build-windows-5.10/Prebuilt-Common.ps1 @@ -0,0 +1,173 @@ +# Shared by the scripts that deal with the prebuilt archives: where they live, and how the C/C++ +# sources they were built from are identified. +# +# The archive name is derived from the content of the sources, not from a version number, so +# there is nothing to bump and nothing to forget: sources that were never built simply have no +# archive, and the build says so. +# +# Nothing here touches the build itself. Build-Mailcore2.ps1 works exactly as it always has; the +# only reason it dot-sources this file is to fetch its binary dependencies from the release +# instead of from S3. + +$Script:MailcorePrebuiltRepo = "readdle/mailcore2" +$Script:MailcorePrebuiltReleaseTag = "windows-prebuilt" +$Script:MailcorePrebuiltUrlBase = "https://github.com/$Script:MailcorePrebuiltRepo/releases/download/$Script:MailcorePrebuiltReleaseTag" + +# The binaries this build links against. It sits in a digested file, so replacing the archive +# changes the digest and every consumer asks for a rebuilt prebuilt - which is right, because +# these three libraries end up inside it. +$Script:MailcoreDependenciesArchive = "mailcore2-windows-deps-1.zip" + +# Everything that determines the content of the prebuilt binaries: the C/C++ sources (Swift is +# compiled from source at build time, so src/swift is excluded), the top-level CMakeLists, and +# this directory - the scripts decide what gets compiled, which project files and redistributables +# get copied in, and what ends up in the package. +# +# Including the scripts costs an occasional rebuild for a change that could not have altered a +# byte - a reworded message, a fix to the pull-request check. That is the safe direction to err +# in: the alternative is an archive whose contents changed under a name that did not. +function Get-MailcoreDigestPathSpec { + return @("src", "CMakeLists.txt", "build-windows-5.10") +} + +# A digest of git's own tree entries, not of file bytes: git already stores a hash per blob, so +# this is instant and, more importantly, identical on every platform - core.autocrlf cannot +# change it, unlike hashing working-tree content. +function Get-MailcoreSourceDigest { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + # Anything git resolves to a tree. Only HEAD can be checked for local edits, so only + # HEAD is; other refs are read straight out of the object store. + [string]$Ref = "HEAD" + ) + + $paths = Get-MailcoreDigestPathSpec + + # src/swift is outside the digest, so a dirty file there must not block it either. + $status = @(& git -C $RepoRoot status --porcelain -- @paths | Where-Object { $_ -notmatch " src/swift/" }) + if ($LASTEXITCODE -ne 0) { throw "Not a git checkout: $RepoRoot" } + if ($Ref -eq "HEAD" -and $status) { + throw "Uncommitted changes under $($paths -join ', '): the digest describes HEAD, so it would not match what is built. Commit them, or build from source with -BuildMailcore2.`n$($status -join "`n")" + } + + $lines = & git -C $RepoRoot ls-tree -r $Ref -- @paths + if ($LASTEXITCODE -ne 0) { throw "git ls-tree failed for $Ref in $RepoRoot" } + $lines = $lines | Where-Object { $_ -notmatch "`tsrc/swift/" } + if (-not $lines) { throw "No source entries found at $Ref in $RepoRoot - wrong directory?" } + + # Canonical listing: git's own order, LF endings, UTF-8 without BOM. Written to a file + # rather than piped, because PowerShell re-encodes text between processes. + $text = ($lines -join "`n") + "`n" + $tempFile = [IO.Path]::GetTempFileName() + try { + [IO.File]::WriteAllText($tempFile, $text, (New-Object Text.UTF8Encoding $false)) + $digest = & git -C $RepoRoot hash-object --no-filters $tempFile + if ($LASTEXITCODE -ne 0) { throw "git hash-object failed" } + } + finally { + Remove-Item -LiteralPath $tempFile -Force -ErrorAction Ignore + } + + return $digest.Trim() +} + +function Get-MailcorePrebuiltArchiveName { + param([Parameter(Mandatory = $true)][string]$Digest) + return "mailcore2-windows-$($Digest.Substring(0, 12)).zip" +} + +# The HTTP status behind a failed Invoke-RestMethod / Invoke-WebRequest, or $null when the +# request never got a response at all - DNS, proxy, a dropped connection. Windows PowerShell +# raises WebException and PowerShell 7 raises HttpResponseException; both expose .Response with +# a .StatusCode, so this works on either. +function Get-MailcoreHttpStatus { + param([Parameter(Mandatory = $true)]$ErrorRecord) + $response = $ErrorRecord.Exception.Response + if (-not $response) { return $null } + try { return [int]$response.StatusCode } catch { return $null } +} + +function Get-MailcorePrebuiltUrl { + param([Parameter(Mandatory = $true)][string]$ArchiveName) + return "$Script:MailcorePrebuiltUrlBase/$ArchiveName" +} + +# The digest of the sources an unpacked archive was built from. +function Get-MailcoreArchiveDigest { + param([Parameter(Mandatory = $true)][string]$UnpackedPath) + $stamp = Join-Path $UnpackedPath "etc\mailcore2-source-digest" + if (-not (Test-Path -LiteralPath $stamp)) { return $null } + return (Get-Content -LiteralPath $stamp -Raw).Trim() +} + +# --- The release -------------------------------------------------------------------------------- + +# The release is a permanent container for binaries, set up once by hand. Nothing here creates it; +# the scripts only add archives to it. +$Script:MailcoreReleaseSetupHelp = @" +The $Script:MailcorePrebuiltReleaseTag release does not exist, or is not visible to this token. +It is created once, by hand: + + 1. https://github.com/$Script:MailcorePrebuiltRepo/releases/new?tag=$Script:MailcorePrebuiltReleaseTag + Tag $Script:MailcorePrebuiltReleaseTag, marked as a pre-release. + 2. Attach the dependency archive to it, under the name Prebuilt-Common.ps1 gives it. +"@ + +function Test-MailcorePrebuiltRelease { + & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json tagName 2>$null | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Assert-MailcorePrebuiltRelease { + if (-not (Test-MailcorePrebuiltRelease)) { throw $Script:MailcoreReleaseSetupHelp } +} + +# Names of every asset on the release, or an empty array when there is no release. Needs gh on +# PATH and authenticated (GH_TOKEN is enough in CI). +function Get-MailcoreReleaseAssetNames { + $assets = & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json assets --jq ".assets[].name" 2>$null + if ($LASTEXITCODE -ne 0) { return @() } + return @($assets) +} + +function Test-MailcoreReleaseAsset { + param([Parameter(Mandatory = $true)][string]$AssetName) + return ((Get-MailcoreReleaseAssetNames) -contains $AssetName) +} + +# --- The dependency archive --------------------------------------------------------------------- + +# openssl, sasl and zlib: the three binaries that used to be fetched from S3, one zip each, with +# SPARK_PREBUILT_KEY. They now travel as one public asset on the release, unchanged, under a +# single top-level mailcore2-windows-deps directory. Nothing else moved: ICU and libxml2 come +# with the toolchain and were never in that bucket. +# +# Returns a local path, downloading the archive when there is not one already. +function Get-MailcoreDependenciesArchive { + param([Parameter(Mandatory = $true)][string]$WorkPath) + + $name = $Script:MailcoreDependenciesArchive + $path = Join-Path $WorkPath $name + if (Test-Path -LiteralPath $path) { return $path } + + New-Item -ItemType Directory -Path $WorkPath -Force | Out-Null + $url = Get-MailcorePrebuiltUrl -ArchiveName $name + Write-Host "Downloading $name" + try { + Invoke-WebRequest -Uri $url -OutFile $path -UseBasicParsing + } + catch { + Remove-Item -LiteralPath $path -Force -ErrorAction Ignore + $status = Get-MailcoreHttpStatus -ErrorRecord $_ + if ($status -ne 404) { + throw "Could not download the dependency archive $name from $url$(if ($status) { " (HTTP $status)" }). $($_.Exception.Message)" + } + throw @" +The dependency archive $name is not on the $Script:MailcorePrebuiltReleaseTag release. + +It carries openssl, sasl and zlib, and is attached by hand, once. Attach it, or drop a copy at +$path and it will be used as is. +"@ + } + return $path +} diff --git a/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 new file mode 100644 index 000000000..4d4f44129 --- /dev/null +++ b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 @@ -0,0 +1,173 @@ +Param( + [string]$WorkPath, + # Rebuild and overwrite an archive that is already published. Same digest means the same + # sources, so this replaces like with like. + [switch]$Force, + [switch]$SkipUpload +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" +if (-Not $WorkPath) { + $WorkPath = "$ProjectRoot\.build\prebuilt" +} + +function Assert-GitHubCli { + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + throw "GitHub CLI is required to publish. Install it once with: winget install --id GitHub.cli" + } + & gh auth status 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "GitHub CLI is not authenticated. Run: gh auth login" + } +} + +# Adds one asset to the release, which is set up once by hand and never created here. +function Publish-ReleaseAsset { + param([Parameter(Mandatory = $true)][string]$Path, [switch]$Clobber) + Assert-MailcorePrebuiltRelease + $uploadArgs = @($Script:MailcorePrebuiltReleaseTag, $Path, "--repo", $Script:MailcorePrebuiltRepo) + if ($Clobber) { $uploadArgs += "--clobber" } + & gh release upload @uploadArgs + if ($LASTEXITCODE -ne 0) { throw "Failed to upload $Path" } +} + +# --- Preflight ------------------------------------------------------------------------------ + +if (-not $SkipUpload) { + Assert-GitHubCli + Assert-MailcorePrebuiltRelease +} + + +# Throws when the digested paths are dirty: the archive must correspond to a committed state. +$Digest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot +$ArchiveName = Get-MailcorePrebuiltArchiveName -Digest $Digest +$GitRev = (& git -C $ProjectRoot rev-parse HEAD).Trim() + +Write-Host "" +Write-Host " source digest : $Digest" -ForegroundColor Cyan +Write-Host " archive : $ArchiveName" -ForegroundColor Cyan +Write-Host " git revision : $GitRev" -ForegroundColor Cyan +Write-Host "" + +$AlreadyPublished = (-not $SkipUpload) -and (Test-MailcoreReleaseAsset -AssetName $ArchiveName) +if ($AlreadyPublished -and -not $Force) { + Write-Host "These sources already have a published prebuilt ($ArchiveName) - nothing to do." -ForegroundColor Green + Write-Host "Pass -Force to rebuild and overwrite it." -ForegroundColor DarkGray + return +} + +# --- Dependency archive --------------------------------------------------------------------- + +$DependenciesPath = "$WorkPath\build-dependencies" +$InstallPath = "$WorkPath\mailcore2-install" +$StagePath = "$WorkPath\mailcore2-all" +$ArchivePath = "$WorkPath\$ArchiveName" + +New-Item -ItemType Directory -Path $WorkPath -Force | Out-Null + +# --- Build ------------------------------------------------------------------------------------ + +# A publish must package only what this build produced. The CMake binary directory matters as +# much as the install tree: src/CMakeLists.txt stages the public headers into it with file(COPY), +# which adds but never prunes, so a header from an earlier build on this machine would be +# packaged even though this revision does not declare it public. +Push-Task -Name "Clean previous build output" -ScriptBlock { + foreach ($stale in $InstallPath, $StagePath, "$ProjectRoot\.build\mailcore2") { + if (Test-Path -LiteralPath $stale) { + Write-TaskLog "Removing $stale" + Remove-Item -LiteralPath $stale -Recurse -Force + } + } +} + +Push-Task -Name "Build mailcore2" -ScriptBlock { + & "$PSScriptRoot\Build-Mailcore2.ps1" ` + -DependenciesPath $DependenciesPath ` + -InstallPath $InstallPath ` + -Install +} + +Push-Task -Name "Stamp source digest" -ScriptBlock { + New-Item -ItemType Directory -Path "$InstallPath\etc" -Force | Out-Null + [IO.File]::WriteAllText("$InstallPath\etc\mailcore2-source-digest", "$Digest`n", (New-Object Text.UTF8Encoding $false)) + + $builtRev = (Get-Content -LiteralPath "$InstallPath\etc\mailcore2-git-rev" -Raw).Trim() + if ($builtRev -ne $GitRev) { + throw "The install tree reports revision $builtRev but the checkout is at $GitRev" + } + } + +Push-Task -Name "Package $ArchiveName" -ScriptBlock { + # The artifact must run without a separately installed VC redistributable. + $vcRedistRoot = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Redist\MSVC" + $vcCrt = Get-ChildItem $vcRedistRoot -Filter msvcp140.dll -File -Recurse | + Where-Object FullName -Match '\\x64\\Microsoft\.VC143\.CRT\\' | + Sort-Object FullName -Descending | + Select-Object -First 1 -ExpandProperty DirectoryName + if (-not $vcCrt) { throw "VC143 x64 redistributable DLLs not found under $vcRedistRoot" } + Copy-Item "$vcCrt\*.dll" (Join-Path $InstallPath "bin") -Force + + # A clean staging directory: copying onto an existing one would nest it on the second run. + if (Test-Path $StagePath) { Remove-Item $StagePath -Recurse -Force } + if (Test-Path $ArchivePath) { Remove-Item $ArchivePath -Force } + Copy-Item $InstallPath $StagePath -Recurse + tar.exe -a -cf $ArchivePath -C $WorkPath mailcore2-all + if ($LASTEXITCODE -ne 0) { throw "Packaging failed" } +} + +Push-Task -Name "Verify $ArchiveName" -ScriptBlock { + $checkDir = "$WorkPath\verify" + if (Test-Path $checkDir) { Remove-Item $checkDir -Recurse -Force } + New-Item -ItemType Directory -Path $checkDir -Force | Out-Null + tar.exe -C $checkDir -xf $ArchivePath + if (-not (Test-Path -LiteralPath "$checkDir\mailcore2-all")) { + throw "The archive must contain exactly one root directory named mailcore2-all" + } + + $unpackedDigest = Get-MailcoreArchiveDigest -UnpackedPath "$checkDir\mailcore2-all" + if ($unpackedDigest -ne $Digest) { + throw "The packaged archive carries digest '$unpackedDigest', expected '$Digest'" + } + + # Direct non-system dependencies: a missing one only shows up at load time otherwise. + $required = @( + "mailcore2.dll", "CMailCore.dll", + "libetpan.dll", "libctemplate.dll", "rdtidy.dll", + "libcrypto-1_1-x64.dll", "libssl-1_1-x64.dll", "zlib.dll", "sasl2.dll", + "msvcp120.dll", "msvcr120.dll" + ) + $missing = $required | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\bin\$_") } + if ($missing) { throw "The archive is missing: $($missing -join ', ')" } + + # A header public-headers.cmake has always declared, so this stays true of older revisions. + $requiredHeaders = @("include\MailCore\MailCore.h") + $missingHeaders = $requiredHeaders | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\$_") } + if ($missingHeaders) { throw "The archive is missing headers: $($missingHeaders -join ', ')" } + + Remove-Item $checkDir -Recurse -Force + Write-TaskLog "Archive verified" +} + +if ($SkipUpload) { + Write-Host "" + Write-Host "Built and verified (upload skipped): $ArchivePath" -ForegroundColor Green + return +} + +Push-Task -Name "Upload $ArchiveName" -ScriptBlock { + Publish-ReleaseAsset -Path $ArchivePath -Clobber:$AlreadyPublished +} + +Write-Host "" +Write-Host "Published $ArchiveName" -ForegroundColor Green +Write-Host " sources : $Digest" -ForegroundColor Green +Write-Host " revision: $GitRev" -ForegroundColor Green +Write-Host " sha256 : $((Get-FileHash $ArchivePath -Algorithm SHA256).Hash)" -ForegroundColor Green +Write-Host "" +Write-Host "Nothing needs to be committed to mailcore2: the archive is named after these sources," -ForegroundColor Green +Write-Host "so any checkout of them finds it. Carry on with the commit, PR and tag as usual." -ForegroundColor Green