From b7d01c04031add93a36d358c7562f6e80a31a277 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 18 Sep 2024 12:29:47 -0700 Subject: [PATCH] Add automated release flow (#394) ## New release workflow Build Helper will now automatically create a draft GitHub Release after it finishes its builds. It will upload a copy of the build artifacts to this release for easy access. When a version is ready to be published, edit the GitHub Release and publish it. This will trigger a workflow to publish the artifacts to our releases feed. ## Moved artifacts scripts to Taskfile The scripts formerly located at `scripts/artifacts` have been moved to the Taskfile. They can now be found at `artifacts:*`. ## Moved releases readme to `RELEASES.md` Updated the releases readme with step-by-step instructions and moved it from `scripts/artifacts` to `RELEASES.md` ## Created new AWS identities for artifact upload and publishing This narrows the scopes of the AWS identities used by the workflows to upload and publish artifacts. The Build Helper workflow now only has permission to put files into the artifacts bucket. The Publish Release workflow only has permission to get files from the artifacts bucket and put them into the releases bucket. --- .github/workflows/build-helper.yml | 29 ++++++-- .github/workflows/bump-version.yml | 2 +- .github/workflows/publish-release.yml | 25 +++++++ .gitignore | 1 + RELEASES.md | 64 +++++++++++++++++ Taskfile.yml | 38 +++++++++- package.json | 2 +- scripts/artifacts/.gitignore | 4 -- scripts/artifacts/README.md | 69 ------------------- scripts/artifacts/download-staged-artifact.sh | 21 ------ scripts/artifacts/publish-from-staging.sh | 25 ------- 11 files changed, 154 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/publish-release.yml create mode 100644 RELEASES.md delete mode 100644 scripts/artifacts/.gitignore delete mode 100644 scripts/artifacts/README.md delete mode 100755 scripts/artifacts/download-staged-artifact.sh delete mode 100755 scripts/artifacts/publish-from-staging.sh diff --git a/.github/workflows/build-helper.yml b/.github/workflows/build-helper.yml index 17461b91..108e0f48 100644 --- a/.github/workflows/build-helper.yml +++ b/.github/workflows/build-helper.yml @@ -3,7 +3,7 @@ # For more information on the Windows Code Signing, see https://docs.digicert.com/en/digicert-keylocker/ci-cd-integrations/plugins/github-custom-action-for-keypair-signing.html and https://docs.digicert.com/en/digicert-keylocker/signing-tools/sign-authenticode-with-electron-builder-using-ksp-integration.html name: Build Helper -run-name: Version ${{ github.ref_name }} +run-name: Build ${{ github.ref_name }} on: push: tags: @@ -13,6 +13,8 @@ env: NODE_VERSION: "22.5.1" jobs: runbuild: + permissions: + contents: write outputs: version: ${{ steps.set-version.outputs.WAVETERM_VERSION }} strategy: @@ -123,8 +125,27 @@ jobs: CSC_KEY_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }} shell: powershell # electron-builder's Windows code signing package has some compatibility issues with pwsh, so we need to use Windows Powershell - name: Upload to S3 staging - run: aws s3 cp make/ s3://waveterm-github-artifacts/staging-w2/${{ steps.set-version.outputs.WAVETERM_VERSION }}/ --recursive --exclude "*/*" --exclude "builder-*.yml" + run: task artifacts:upload env: - AWS_ACCESS_KEY_ID: "${{ secrets.S3_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.S3_KEY_SECRET }}" + AWS_ACCESS_KEY_ID: "${{ secrets.ARTIFACTS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.ARTIFACTS_KEY_SECRET }}" AWS_DEFAULT_REGION: us-west-2 + + - name: Create draft release + uses: softprops/action-gh-release@v2 + with: + prerelease: ${{ contains(github.ref_name, '-beta') }} + name: Wave Terminal ${{ github.ref_name }} Release + generate_release_notes: true + draft: true + files: | + make/*.zip + make/*.dmg + make/*.exe + make/*.msi + make/*.rpm + make/*.deb + make/*.pacman + make/*.snap + make/*.flatpak + make/*.AppImage diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 665dbd65..fd6cd78d 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -3,7 +3,7 @@ # For more information, see this doc: https://github.com/Nautilus-Cyberneering/pygithub/blob/main/docs/how_to_sign_automatic_commits_in_github_actions.md name: Bump Version -run-name: "branch: ${{ github.ref_name }}; version: ${{ inputs.bump }}; prerelease: ${{ inputs.is-prerelease }}" +run-name: "branch: ${{ github.ref_name }}; semver-bump: ${{ inputs.bump }}; prerelease: ${{ inputs.is-prerelease }}" on: workflow_dispatch: inputs: diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 00000000..3d7fdc97 --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,25 @@ +# Workflow to copy artifacts from the staging bucket to the release bucket when a new GitHub Release is published. + +name: Publish Release +run-name: Publish ${{ github.ref_name }} +on: + release: + types: [published] +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Publish from staging + if: startsWith(github.ref, 'refs/tags/') + run: "task artifacts:publish:${{ github.ref_name }}" + env: + AWS_ACCESS_KEY_ID: "${{ secrets.PUBLISHER_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.PUBLISHER_KEY_SECRET }}" + AWS_DEFAULT_REGION: us-west-2 + shell: bash diff --git a/.gitignore b/.gitignore index 1c3fc393..ec8952a1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ bin/ *~ out/ make/ +artifacts/ # Yarn Modern .pnp.* diff --git a/RELEASES.md b/RELEASES.md new file mode 100644 index 00000000..2bc7dc91 --- /dev/null +++ b/RELEASES.md @@ -0,0 +1,64 @@ +# Building for release + +## Step-by-step guide + +1. Go to the [Actions tab](https://github.com/wavetermdev/thenextwave/actions) and select "Bump Version" from the left sidebar. +2. Click on "Run workflow". You will see two options: + - "SemVer Bump": This defaults to `none`. Adjust this if you want to increment the version number according to semantic versioning rules (`patch`, `minor`, `major`). + - "Is Prerelease": This defaults to `true`. If set to `true`, a `-beta.X` version will be appended to the end of the version. If one is already present and the base SemVer is not being incremented, the `-beta` version will be incremented (i.e. `0.1.13-beta.0` to `0.1.13-beta.1`). +3. After "Bump Version" a "Build Helper" run will kick off automatically for the new version. When this completes, it will generate a draft GitHub Release with all the built artifacts. +4. Review the artifacts in the release and test them locally. +5. When you are confident that the build is good, edit the GitHub Release to add a changelog and release summary and publish the release. +6. The new version will be published to our release feed automatically when the GitHub Release is published. If the build is a prerelease, it will only release to users subscribed to the `beta` channel. If it is a general release, it will be released to all users. + +## Details + +### Bump Version workflow + +All releases start by first bumping the package version and creating a new Git tag. We have a workflow set up to automate this. + +To run it, trigger a new run of the [Bump Version workflow](https://github.com/wavetermdev/thenextwave/actions/workflows/bump-version.yml). When triggering the run, you will be prompted to select a version bump type, either `none`, `patch`, `minor`, or `major`, and whether the version is prerelease or not. This determines how much the version number is incremented. + +See [`version.cjs`](../../version.cjs) for more details on how this works. + +Once the tag has been created, a new [Build Helper](#build-helper-workflow) run will be automatically queued to generate the artifacts. + +### Build Helper workflow + +Our release builds are managed by the [Build Helper workflow](https://github.com/wavetermdev/thenextwave/actions/workflows/build-helper.yml). + +Under the hood, this will call the `package` task in [`Taskfile.yml`](../../Taskfile.yml), which will build the `wavesrv` and `wsh` binaries, then the frontend and Electron codebases using Vite, then it will call `electron-builder` to generate the distributable app packages. The configuration for `electron-builder` is defined in [`electron-builder.config.cjs`](../../electron-builder.config.cjs). + +This will also sign and notarize the macOS app package. + +Once a build is complete, it will be placed in `s3://waveterm-github-artifacts/staging-w2/`. It can be downloaded for testing using the `artifacts:download:*` task. When you are ready to publish the artifacts to the public release feed, use the `artifacts:publish:*` task to directly copy the artifacts from the staging bucket to the releases bucket. + +You will need to configure an AWS CLI profile with write permissions for the S3 buckets in order for the script to work. You should invoke the tasks as follows: + +```bash +task artifacts:: -- --profile +``` + +### Automatic updates + +Thanks to [`electron-updater`](https://www.electron.build/auto-update.html), we are able to provide automatic app updates for macOS, Linux, and Windows, as long as the app was distributed as a DMG, AppImage, RPM, or DEB file (all Windows targets support auto updates). + +With each release, YAML files will be produced that point to the newest release for the current channel. These also include file sizes and checksums to aid in validating the packages. The app will check these files in our S3 bucket every hour to see if a new version is available. + +#### Update channels + +We utilize update channels to roll out beta and stable releases. These are determined based on the package versioning [described above](#bump-version-workflow). Users can select their update channel using the `autoupdate:channel` setting in Wave. See [here](https://www.electron.build/tutorials/release-using-channels.html) for more information. + +#### Homebrew + +Homebrew is automatically bumped when new artifacts are published. + +#### Linux + +We do not currently submit the Linux packages to any of the package repositories. We are working on addressing this in the near future. + +### `electron-build` configuration + +Most of our configuration is fairly standard. The main exception to this is that we exclude our Go binaries from the ASAR archive that Electron generates. ASAR files cannot be executed by NodeJS because they are not seen as files and therefore cannot be executed via a Shell command. More information can be found [here](https://www.electronjs.org/docs/latest/tutorial/asar-archives#executing-binaries-inside-asar-archive). + +We also exclude most of our `node_modules` from packaging, as Vite handles packaging of any dependencies for us. The one exception is `monaco-editor`. diff --git a/Taskfile.yml b/Taskfile.yml index 998f2367..8e488482 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -11,6 +11,8 @@ vars: RM: '{{if eq OS "windows"}}cmd --% /c del /S{{else}}rm {{end}}' RMRF: '{{if eq OS "windows"}}powershell Remove-Item -Force -Recurse{{else}}rm -rf{{end}}' DATE: '{{if eq OS "windows"}}powershell Get-Date -UFormat{{else}}date{{end}}' + ARTIFACTS_BUCKET: waveterm-github-artifacts/staging-w2 + RELEASES_BUCKET: dl.waveterm.dev/releases-w2 tasks: electron:dev: @@ -201,7 +203,7 @@ tasks: - frontend/app/store/wshserver.ts version: - desc: Get the current package version, or bump version if args are present. + desc: Get the current package version, or bump version if args are present. To pass args to `version.cjs`, add them after `--`. summary: | If no arguments are present, the current version will be returned. If only a single argument is given, the following are valid inputs: @@ -213,6 +215,40 @@ tasks: If two arguments are given, the first argument must be either `none`, `patch`, `minor`, or `major`. The second argument must be `1` or `true` to bump the prerelease version. cmd: node version.cjs {{.CLI_ARGS}} + artifacts:upload: + desc: Uploads build artifacts to the staging bucket in S3. To add additional AWS CLI arguments, add them after `--`. + vars: + ORIGIN: "make/" + DESTINATION: "{{.ARTIFACTS_BUCKET}}/{{.VERSION}}" + cmd: aws s3 cp {{.ORIGIN}}/ s3://{{.DESTINATION}}/ --recursive --exclude "*/*" --exclude "builder-*.yml" {{.CLI_ARGS}} + + artifacts:download:*: + desc: Downloads the specified artifacts version from the staging bucket. To add additional AWS CLI arguments, add them after `--`. + vars: + DL_VERSION: '{{ replace "v" "" (index .MATCH 0)}}' + ORIGIN: "{{.ARTIFACTS_BUCKET}}/{{.DL_VERSION}}" + DESTINATION: "artifacts/{{.DL_VERSION}}" + cmds: + - '{{.RMRF}} "{{.DESTINATION}}"' + - aws s3 cp s3://{{.ORIGIN}}/ {{.DESTINATION}}/ --recursive {{.CLI_ARGS}} + + artifacts:publish:*: + desc: Publishes the specified artifacts version from the staging bucket to the releases bucket. To add additional AWS CLI arguments, add them after `--`. + vars: + UP_VERSION: '{{ replace "v" "" (index .MATCH 0)}}' + ORIGIN: "{{.ARTIFACTS_BUCKET}}/{{.UP_VERSION}}" + DESTINATION: "{{.RELEASES_BUCKET}}" + cmd: | + OUTPUT=$(aws s3 cp s3://{{.ORIGIN}}/ s3://{{.DESTINATION}}/ --recursive {{.CLI_ARGS}}) + + for line in $OUTPUT; do + PREFIX=${line%%{{.DESTINATION}}*} + SUFFIX=${line:${#PREFIX}} + if [[ -n "$SUFFIX" ]]; then + echo "https://$SUFFIX" + fi + done + yarn: desc: Runs `yarn` internal: true diff --git a/package.json b/package.json index ee508055..936bb7c5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "productName": "TheNextWave", "description": "An Open-Source, AI-Native, Terminal Built for Seamless Workflows", "license": "Apache-2.0", - "version": "0.1.13-beta.26", + "version": "0.1.13-beta.37", "homepage": "https://waveterm.dev", "build": { "appId": "dev.commandline.thenextwave" diff --git a/scripts/artifacts/.gitignore b/scripts/artifacts/.gitignore deleted file mode 100644 index a3e60d5b..00000000 --- a/scripts/artifacts/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*builds/ -*-staged/ -*.zip -*.dmg diff --git a/scripts/artifacts/README.md b/scripts/artifacts/README.md deleted file mode 100644 index 58796d81..00000000 --- a/scripts/artifacts/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Building for release - -## Bump Version workflow - -All releases start by first bumping the package version and creating a new Git tag. We have a workflow set up to automate this. - -To run it, trigger a new run of the [Bump Version workflow](https://github.com/wavetermdev/thenextwave/actions/workflows/bump-version.yml). When triggering the run, you will be -prompted to select a version bump type, either `none`, `patch`, `minor`, or `major`, and whether the version is prerelease or not. This determines how much the version number is incremented. -See [`version.cjs`](../../version.cjs) for more details on how this works. - -Once the tag has been created, a new [Build Helper](#build-helper-workflow) run will be automatically queued to generate the artifacts. - -## Build Helper workflow - -Our release builds are managed by the [Build Helper workflow](https://github.com/wavetermdev/thenextwave/actions/workflows/build-helper.yml). - -Under the hood, this will call the `package` task in -[`Taskfile.yml`](../../Taskfile.yml), which will build the Electron codebase using -WebPack and then the `wavesrv` and `mshell` binaries, then it will call `electron-builder` -to generate the distributable app packages. The configuration for `electron-builder` -is [`electron-builder.config.cjs`](../../electron-builder.config.cjs). - -This will also sign and notarize the macOS app package. - -Once a build is complete, it will be placed in `s3://waveterm-github-artifacts/staging-w2/`. -It can be downloaded for testing using the [`download-staged-artifact.sh`](./download-staged-artifact.sh) -script. When you are ready to publish the artifacts to the public release feed, use the -[`publish-from-staging.sh`](./publish-from-staging.sh) script to directly copy the artifacts from -the staging bucket to the releases bucket. - -You will need to configure an AWS CLI profile with write permissions for the S3 buckets in order for the script to work. You should invoke the script as follows: - -```bash -