diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f404648..88e292f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,13 +1,19 @@ name: Release # Releasing is a version bump: push a commit to main that changes the version in -# package.json, and this tags v, creates the GitHub release and -# publishes to npm. Every other push to main only runs the tests. +# package.json, and this tags v, creates the GitHub release and stages +# the npm publish. Every other push to main only runs the tests. # -# Running it by hand instead publishes whatever version package.json holds, as -# long as npm does not have it yet, and skips the tag and the release if a -# previous attempt already made them. That is how a half-finished release is -# retried. +# Staging is not publishing: "npm stage publish" hands npm the tarball and nobody +# can install it until a maintainer runs "npm stage approve " and +# answers a 2FA prompt. That is the point — npm is deprecating the tokens that +# let a workflow publish on its own, and staging defers the proof of presence +# instead of removing it. The job summary carries the command to run. +# +# Running it by hand instead stages whatever version package.json holds, as long +# as npm has neither published nor staged it yet, and skips the tag and the +# release if a previous attempt already made them. That is how a half-finished +# release is retried. on: push: branches: [main] @@ -48,12 +54,14 @@ jobs: echo "tag_exists=$tag_exists" >> "$GITHUB_OUTPUT" if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - # by hand: publish what package.json says, unless npm has it already + # by hand: stage what package.json says, unless npm has it already. + # A version staged but not yet approved is invisible to npm view; the + # stage job looks for that one itself. if npm view "$name@$version" version >/dev/null 2>&1; then - echo "::warning::$name@$version is already on npm; nothing to publish" + echo "::warning::$name@$version is already on npm; nothing to stage" echo "changed=false" >> "$GITHUB_OUTPUT" else - echo "publishing $name@$version by hand (tag exists: $tag_exists)" + echo "staging $name@$version by hand (tag exists: $tag_exists)" echo "changed=true" >> "$GITHUB_OUTPUT" fi exit 0 @@ -113,13 +121,13 @@ jobs: if-no-files-found: error publish: - name: Tag and publish + name: Tag and stage needs: [version, prebuild] if: needs.version.outputs.changed == 'true' runs-on: ubuntu-latest permissions: contents: write # the tag and the release - id-token: write # provenance for npm + id-token: write # the OIDC exchange, and provenance for npm env: VERSION: ${{ needs.version.outputs.version }} steps: @@ -161,7 +169,74 @@ jobs: else gh release create "v$VERSION" --title "v$VERSION" --generate-notes tree-sitter-django.wasm fi - - name: Publish + # "npm stage" arrived in npm 12, which is newer than the npm any runner + # image ships with + - name: Install npm 12 + run: npm install -g npm@^12 + # staged versions share npm's version index with published ones, so a + # second stage of the same version is an error rather than a no-op. A run + # that staged and then failed later should find its stage and go on. + - name: Look for an existing stage + id: staged env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --provenance --access public + run: | + set -uo pipefail + name="$(node -p "require('./package.json').name")" + # a short-lived OIDC token cannot run "npm stage list" at all, so a + # failure here means "ask npm by staging", not "stop" + if ! items="$(npm stage list "$name" --json 2>/dev/null)"; then + echo "could not list staged versions; assuming there are none" + echo "staged=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + id="$(node -e ' + const items = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + const match = items.find((item) => item.version === process.env.VERSION); + if (match) console.log(match.id); + ' <<< "$items")" + if [ -n "$id" ]; then + echo "::warning::$name@$VERSION is already staged as $id; not staging it again" + echo "staged=true" >> "$GITHUB_OUTPUT" + echo "stage_id=$id" >> "$GITHUB_OUTPUT" + else + echo "staged=false" >> "$GITHUB_OUTPUT" + fi + # NPM_TOKEN is the fallback: where the package has a trusted publisher for + # this workflow, npm exchanges the OIDC token above for one of its own and + # the secret is not read at all. + - name: Stage the publish + id: stage + if: steps.staged.outputs.staged != 'true' + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + npm stage publish --provenance --access public 2>&1 | tee stage.log + id="$(sed -n 's/.*staged with id \([0-9a-fA-F-]\{36\}\).*/\1/p' stage.log | tail -n1)" + echo "stage_id=$id" >> "$GITHUB_OUTPUT" + - name: Say how to approve it + env: + STAGE_ID: ${{ steps.stage.outputs.stage_id || steps.staged.outputs.stage_id }} + run: | + set -euo pipefail + name="$(node -p "require('./package.json').name")" + { + echo "### $name@$VERSION is staged, not published" + echo + echo "npm has the tarball and nobody can install it until a maintainer approves it, which asks for 2FA:" + echo + echo '```sh' + if [ -n "$STAGE_ID" ]; then + echo "npm stage view $STAGE_ID # what was staged" + echo "npm stage download $STAGE_ID # the tarball itself" + echo "npm stage approve $STAGE_ID # publish it" + echo "npm stage reject $STAGE_ID # throw it away" + else + echo "npm stage list $name" + echo "npm stage approve " + fi + echo '```' + echo + echo "Approving needs npm 12: \`npm install -g npm@^12\`." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 23f4c69..f93145d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ libtree-sitter-django.* .build/ Package.resolved *.pc +*.tgz diff --git a/CLAUDE.md b/CLAUDE.md index 95648ba..9acf71b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,19 +64,33 @@ and the Rust, Python, Go and Swift bindings. `.github/workflows/release.yml` runs on every push to `main` but only releases when `package.json`'s version differs from the previous commit's and no `v` tag exists yet. Then it runs CI, `prebuildify` on five runners (Linux x64 and arm64, macOS x64 and arm64, Windows x64), and a last job that tags `v`, -creates the GitHub release with `tree-sitter-django.wasm` attached, and runs `npm publish --provenance`. -**A release is a version-bump commit on `main`**; no other push starts one. +creates the GitHub release with `tree-sitter-django.wasm` attached, and runs +`npm stage publish --provenance`. **A release is a version-bump commit on `main`**; no other push starts one. -Running the workflow by hand (`workflow_dispatch`) is the retry: it publishes whatever version -`package.json` holds as long as `npm view` does not find it on the registry already, and it skips the tag -and the release when a previous attempt left them behind, replacing only the Wasm asset. So a run that -tagged and then failed to publish is finished by running it again, and a run against an already published -version stops at the version job. +**The workflow stages the publish; it does not publish.** `npm stage publish` hands npm the tarball and +holds it there, unavailable to install, until a maintainer runs `npm stage approve ` and answers a +2FA prompt — proof of presence deferred rather than dropped, which is npm's answer to deprecating the tokens +that let a workflow publish unattended. The job summary prints the `view`/`download`/`approve`/`reject` +commands for the stage it made. Approving needs npm 12 (`npm install -g npm@^12`), which is also why the job +installs it: no runner image ships an npm that has the command. + +Running the workflow by hand (`workflow_dispatch`) is the retry: it stages whatever version `package.json` +holds as long as `npm view` does not find it on the registry already, and it skips the tag and the release +when a previous attempt left them behind, replacing only the Wasm asset. So a run that tagged and then +failed to stage is finished by running it again, and a run against an already published version stops at the +version job. A version that is _staged_ and not yet approved is invisible to `npm view` while still +occupying npm's version index, so a second stage of it is an error rather than a no-op; the stage job asks +`npm stage list` first and skips straight to the approval summary when it finds one. That listing is +best-effort — a short-lived OIDC token may not run `npm stage` subcommands at all — and a failure there just +means the stage is attempted. Releasing needs three things to line up: -- the `NPM_TOKEN` secret, a granular automation token. Provenance also needs `id-token: write`, which the - publish job already declares. +- credentials to stage with: either a trusted publisher configured for this workflow + (`npm trust github --allow-stage-publish`), which needs no secret at all because npm exchanges the + workflow's OIDC token for one of its own, or the `NPM_TOKEN` secret as the fallback. Staging works with any + token type, so that token no longer needs — and should no longer have — the 2FA bypass an automation token + carries. Both the exchange and provenance need `id-token: write`, which the publish job already declares. - **the version in `package.json` and in `tree-sitter.json`'s `metadata`**. The release trigger reads the first; the second is compiled into `src/parser.c` as the language's version, so bumping one alone leaves the parser disagreeing with the manifest. @@ -86,6 +100,21 @@ Releasing needs three things to line up: The arm64 Linux runner (`ubuntu-24.04-arm`) is free for public repositories only. `tree-sitter build --wasm` runs Emscripten through Docker, which the ubuntu runners provide. +**The npm package must carry its prebuilds**, or `npm install` runs `node-gyp-build`, finds nothing and +compiles, which fails on an image without `python3`, `make` and a compiler — `node:20-slim`, for instance. +0.1.0 was published by hand before the workflow existed and shipped none, which is what that image hit. The +`prebuild` jobs are what produce them, and `files` carries `prebuilds/**`. + +`npm run prebuildify` passes `--napi --strip --tag-libc`. `--napi` makes one binary per platform that every +Node ABI can load, so the file needs no ABI tag and is named after the package. `--tag-libc` adds `glibc` or +`musl`: `node-gyp-build` treats a file with no libc tag as matching anything, so an Alpine install would +otherwise load a glibc binary and fail at run time; with the tag it compiles instead. Only glibc is built, +so musl installs need a toolchain. + +`package.json` exports the Wasm build as `tree-sitter-django/tree-sitter-django.wasm`. An `exports` map +hides everything it does not name, so each published path — the queries, `grammar.js`, `src/scanner.c` — +needs its own entry. + ### Stale parser library tree-sitter caches the compiled parser in `~/.cache/tree-sitter/lib/django.*`, and it has served a stale diff --git a/README.md b/README.md index f728709..7a1f47d 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,15 @@ disagree is listed under [Divergences](#divergences-from-django). ## Using it -There is no published package yet, but the language bindings are in place. What is here today: +`tree-sitter-django` is [on npm](https://www.npmjs.com/package/tree-sitter-django), and the other language +bindings are in place. What is here today: - **`src/parser.c` and `src/scanner.c` are generated and committed**, so any tree-sitter host can compile them directly without running the CLI. The scanner is required: block-name matching (`{% block a %}…{% endblock a %}`), the repeated-argument guards and the missing-tag markers live in it. -- **`npm run parser-build`** produces `tree-sitter-django.wasm` for a `web-tree-sitter` host or an editor - that loads Wasm grammars. It is a build artifact rather than a checked-in one, so it has to be built - (or shipped by a release) before anything can load it. +- **`tree-sitter-django.wasm`** is for a `web-tree-sitter` host or an editor that loads Wasm grammars. It + is not checked in: `npm run parser-build` makes it, and a release ships it both in the npm package (the + `tree-sitter-django/tree-sitter-django.wasm` export) and as an asset on the GitHub release. - **`queries/`** holds the highlight, locals, tags, injection and error queries an editor loads, plus three files meant for a linter. See [Queries](#queries). - **`tree-sitter.json`** names the grammar `django`, scopes it `source.django`, and points at the query @@ -35,14 +36,14 @@ There is no published package yet, but the language bindings are in place. What ## Bindings -| Language | Build from a checkout | Entry point | -| -------- | -------------------------------------------------------- | ------------------------------------------------------ | -| Node | `npm install` builds the addon through `node-gyp-build` | `bindings/node/index.js` (ESM), typed by `index.d.ts` | -| Rust | `cargo build` | `bindings/rust/lib.rs`, `Cargo.toml` | -| Python | `pip install .` | `bindings/python/tree_sitter_django`, `pyproject.toml` | -| Go | `go test ./bindings/go` | `bindings/go/binding.go`, `go.mod` | -| Swift | `swift build` | `bindings/swift/TreeSitterDjango`, `Package.swift` | -| C / C++ | `make` (static, shared and a pkg-config file) or `cmake` | `bindings/c/tree_sitter/tree-sitter-django.h` | +| Language | Build from a checkout | Entry point | +| -------- | ------------------------------------------------------------- | ------------------------------------------------------ | +| Node | `npm install`, which uses a prebuilt binary when there is one | `bindings/node/index.js` (ESM), typed by `index.d.ts` | +| Rust | `cargo build` | `bindings/rust/lib.rs`, `Cargo.toml` | +| Python | `pip install .` | `bindings/python/tree_sitter_django`, `pyproject.toml` | +| Go | `go test ./bindings/go` | `bindings/go/binding.go`, `go.mod` | +| Swift | `swift build` | `bindings/swift/TreeSitterDjango`, `Package.swift` | +| C / C++ | `make` (static, shared and a pkg-config file) or `cmake` | `bindings/c/tree_sitter/tree-sitter-django.h` | ```js import Parser from "tree-sitter"; @@ -52,6 +53,17 @@ const parser = new Parser(); parser.setLanguage(django); ``` +To use the parser on a website, the wasm needs to be served then loaded. + +```js +import { Language, Parser } from "web-tree-sitter"; + +await Parser.init(); +const wasm = "my-resources-path/tree-sitter-django.wasm"; +const parser = new Parser(); +parser.setLanguage(await Language.load(wasm)); +``` + ```python import tree_sitter, tree_sitter_django @@ -76,10 +88,12 @@ that pass against the wrong grammar. Pull requests run the corpus tests and every binding through `.github/workflows/ci.yml`. A release is a version-bump commit on `main`: raise the version in `package.json` and in `tree-sitter.json`'s `metadata`, -run `npm run parser-generate`, and push. The workflow tags `v`, publishes to npm with provenance -and prebuilt binaries for Linux, macOS and Windows, and attaches `tree-sitter-django.wasm` to the GitHub -release. Running the workflow by hand publishes the current version if npm does not have it yet, -which is how a release that failed part way through is retried. +run `npm run parser-generate`, and push. The workflow tags `v`, attaches +`tree-sitter-django.wasm` to the GitHub release, and stages the npm publish with provenance and prebuilt +binaries for Linux, macOS and Windows. Staging is not publishing: npm holds the tarball until a maintainer +runs `npm stage approve ` with 2FA, and the workflow's job summary carries that command. Running +the workflow by hand stages the current version if npm does not have it yet, which is how a release that +failed part way through is retried. ## Queries diff --git a/package.json b/package.json index 6850535..3596e00 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tree-sitter-django", - "version": "0.1.0", + "version": "0.1.1", "license": "MIT", "type": "module", "exports": { @@ -9,6 +9,7 @@ "default": "./bindings/node/index.js" }, "./grammar": "./grammar.js", + "./tree-sitter-django.wasm": "./tree-sitter-django.wasm", "./src/scanner.c": "./src/scanner.c", "./queries/*": "./queries/*", "./package.json": "./package.json" @@ -25,7 +26,7 @@ "tag": "tree-sitter tag", "playground": "tree-sitter playground", "install": "node-gyp-build", - "prebuildify": "prebuildify --napi --strip", + "prebuildify": "prebuildify --napi --strip --tag-libc", "binding-test": "node --test bindings/node/*_test.js" }, "dependencies": { diff --git a/src/parser.c b/src/parser.c index e69682c..d87a333 100644 --- a/src/parser.c +++ b/src/parser.c @@ -64580,7 +64580,7 @@ TS_PUBLIC const TSLanguage *tree_sitter_django(void) { .metadata = { .major_version = 0, .minor_version = 1, - .patch_version = 0, + .patch_version = 1, }, }; return &language; diff --git a/tree-sitter.json b/tree-sitter.json index 71f0755..8c62b8a 100644 --- a/tree-sitter.json +++ b/tree-sitter.json @@ -19,7 +19,7 @@ } ], "metadata": { - "version": "0.1.0", + "version": "0.1.1", "license": "MIT", "description": "Django grammar for tree-sitter", "authors": [