{"data":[{"slug":"invoice-coding-eval-1-0","title":"invoice-coding-eval 1.0","evalSlug":"invoice-coding-eval","evalName":"invoice-coding-eval","version":"1.0","commitDate":"2026-10-28T06:58:21.000Z","current":true,"versions":[{"slug":"invoice-coding-eval-1-0","title":"invoice-coding-eval 1.0","version":"1.0","commitSha":"e2406af279d65346a2d6ff6c543860f1cd5c00e9","commitDate":"2026-10-28T06:58:21.000Z","approvedAt":"2026-10-28T06:58:46.017Z","current":true}],"repo":"kstrat2001/invoice-coding-repo","commitSha":"e2406af279d65346a2d6ff6c543860f1cd5c00e9","manifestHash":"f022bfdbd1e142e3d1192a083c6db82415045a1adeab4d93fbe78f0253145c8e","approvedAt":"2026-10-28T06:58:46.017Z","demo":false,"provisionalNote":null,"summary":null,"runs":39,"referenceRunId":"-oSCAN7LzArs","criteria":null,"kind":"structured-output","language":null},{"slug":"biz-scrape-fixture-1-2","title":"biz-scrape-fixture 1.2","evalSlug":"biz-scrape-fixture","evalName":"biz-scrape-fixture","version":"1.2","commitDate":"2026-10-28T06:52:29.000Z","current":true,"versions":[{"slug":"biz-scrape-fixture-1-2","title":"biz-scrape-fixture 1.2","version":"1.2","commitSha":"adc974eb08eb6718cd5fa7db1f695e10310deacb","commitDate":"2026-10-28T06:52:29.000Z","approvedAt":"2026-10-28T08:45:00.239Z","current":true},{"slug":"biz-scrape-fixture-1-1","title":"biz-scrape-fixture 1.1","version":"1.1","commitSha":"701cf6a1860601ad84c624602f888e3fb3743d39","commitDate":null,"approvedAt":"2026-10-24T03:53:45.802Z","current":false}],"repo":"kstrat2001/biz-scrape-fixture","commitSha":"adc974eb08eb6718cd5fa7db1f695e10310deacb","manifestHash":"33b29bfe30bcf425be5497866566553df4279f82fff2bcb07b7155f27ddf036e","approvedAt":"2026-10-28T08:45:00.239Z","demo":false,"provisionalNote":null,"summary":null,"runs":13,"referenceRunId":"shjUGh-FoLzS","criteria":null,"kind":"structured-output","language":null},{"slug":"path-traversal-eval-1-3","title":"path-traversal-eval 1.3","evalSlug":"path-traversal-eval","evalName":"path-traversal-eval","version":"1.3","commitDate":"2026-10-28T06:52:21.000Z","current":true,"versions":[{"slug":"path-traversal-eval-1-3","title":"path-traversal-eval 1.3","version":"1.3","commitSha":"4dda90861aa12178d2779ef6b52817c2d08b2235","commitDate":"2026-10-28T06:52:21.000Z","approvedAt":"2026-10-28T08:44:58.036Z","current":true},{"slug":"path-traversal-eval-1-2","title":"path-traversal-eval 1.2","version":"1.2","commitSha":"6fad7a020d3ad01442ab516c6e7b59202e427a12","commitDate":"2026-10-27T13:43:17.000Z","approvedAt":"2026-10-27T14:33:50.598Z","current":false},{"slug":"path-traversal-eval-1-1","title":"path-traversal-eval 1.1","version":"1.1","commitSha":"7f6903038a52745352882cb44397a862ab568c1e","commitDate":null,"approvedAt":"2026-10-25T15:24:21.588Z","current":false}],"repo":"kstrat2001/path-traversal-eval","commitSha":"4dda90861aa12178d2779ef6b52817c2d08b2235","manifestHash":"b4a3bfb70879c284e3270a5270ce8ae111b5ba112653b88b01718ba61596b40f","approvedAt":"2026-10-28T08:44:58.036Z","demo":false,"provisionalNote":null,"summary":"An app stores files in one folder and lets users name the file they want: a download link, an upload, a delete button. If the app joins that name onto the folder without checking where the result…","runs":12,"referenceRunId":"0uzQItxDC8E5","criteria":[{"id":"download","name":"Download refuses outside names","description":"read_file refuses a name that resolves outside the storage root.","path":"files_app/storage.py","lineStart":39,"lineEnd":39,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L39"},{"id":"upload","name":"Upload refuses outside names","description":"save_file refuses a name that resolves outside the storage root and writes nothing.","path":"files_app/storage.py","lineStart":44,"lineEnd":44,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L44"},{"id":"delete","name":"Delete refuses outside names","description":"delete_file refuses a name that resolves outside the storage root and deletes nothing.","path":"files_app/storage.py","lineStart":51,"lineEnd":51,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L51"},{"id":"folder-listing","name":"Listing refuses outside folders","description":"list_dir refuses a folder name that resolves outside the storage root.","path":"files_app/storage.py","lineStart":56,"lineEnd":56,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L56"},{"id":"archive-extraction","name":"Archive extraction all or nothing","description":"extract_archive refuses an archive with any member that would land outside the root, and writes nothing from it.","path":"files_app/storage.py","lineStart":71,"lineEnd":71,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L71"},{"id":"sibling-folder","name":"Sibling folder is outside","description":"A sibling folder whose name merely starts with the root's name is outside the root and refused.","path":"files_app/storage.py","lineStart":34,"lineEnd":34,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L34"},{"id":"symlinks","name":"Symlinks resolved first","description":"A symlink inside the root that points outside it is refused.","path":"files_app/storage.py","lineStart":33,"lineEnd":33,"url":"https://github.com/kstrat2001/path-traversal-eval/blob/4dda90861aa12178d2779ef6b52817c2d08b2235/files_app/storage.py#L33"}],"kind":"coding","language":"python"},{"slug":"idor-check-eval-1-3","title":"idor-check-eval 1.3","evalSlug":"idor-check-eval","evalName":"idor-check-eval","version":"1.3","commitDate":"2026-10-28T06:52:13.000Z","current":true,"versions":[{"slug":"idor-check-eval-1-3","title":"idor-check-eval 1.3","version":"1.3","commitSha":"e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f","commitDate":"2026-10-28T06:52:13.000Z","approvedAt":"2026-10-28T08:44:55.982Z","current":true},{"slug":"idor-check-eval-1-2","title":"idor-check-eval 1.2","version":"1.2","commitSha":"1146ce262de8a59a06ec25e155bebd4c715ef3ba","commitDate":"2026-10-27T13:43:33.000Z","approvedAt":"2026-10-27T14:33:53.139Z","current":false},{"slug":"idor-check-eval-1-1","title":"idor-check-eval 1.1","version":"1.1","commitSha":"2b18f370f33e597fdb507120195d529db5329e9e","commitDate":null,"approvedAt":"2026-10-25T15:24:22.870Z","current":false}],"repo":"kstrat2001/idor-check-eval","commitSha":"e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f","manifestHash":"f55925c8aa1defa9d2f8f4d632d5608e6536020b9d27f090b84b55ba9eff8ab1","approvedAt":"2026-10-28T08:44:55.982Z","demo":false,"provisionalNote":null,"summary":"An app checks that you are signed in, but not that the record you asked for is yours. Change the id in the link and you get another customer's data. Security people call this an insecure direct…","runs":17,"referenceRunId":"obETRM1iDEgn","criteria":[{"id":"share-in-org","name":"Sharing stays in the organization","description":"Sharing with a user in another organization raises Forbidden, and with an unknown user NotFound.","path":"eval/hidden/test_hidden_sharing.py","lineStart":82,"lineEnd":82,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L82"},{"id":"revoke-immediate","name":"Revoke takes effect at once","description":"After a revoke, the very next get or list by that user behaves as if the document had never been shared.","path":"eval/hidden/test_hidden_sharing.py","lineStart":68,"lineEnd":68,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L68"},{"id":"owner-only-sharing","name":"Only the owner shares","description":"Only the owner can share or revoke: a reader gets Forbidden, anyone who cannot read the document gets NotFound.","path":"eval/hidden/test_hidden_sharing.py","lineStart":93,"lineEnd":93,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L93"},{"id":"shared-user-reads","name":"Shared user can read","description":"A user the document is shared with can get it, and it appears in their list.","path":"eval/hidden/test_hidden_sharing.py","lineStart":46,"lineEnd":46,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L46"},{"id":"shared-read-only","name":"Sharing is read-only","description":"A user the document is shared with gets Forbidden on update and delete.","path":"eval/hidden/test_hidden_sharing.py","lineStart":55,"lineEnd":55,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L55"},{"id":"admin-scope","name":"Admin reads only in own org","description":"An admin reads every document in their own organization, cannot update or delete others' documents, and sees nothing in another organization.","path":"eval/hidden/test_hidden_sharing.py","lineStart":116,"lineEnd":116,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L116"},{"id":"no-existence-leak","name":"No existence leak","description":"A document the caller cannot read looks exactly like one that does not exist, in get and in list.","path":"eval/hidden/test_hidden_sharing.py","lineStart":133,"lineEnd":133,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L133"},{"id":"owner-keeps-access","name":"Owner keeps full access","description":"Everything the owner could do before still works.","path":"eval/hidden/test_hidden_sharing.py","lineStart":37,"lineEnd":37,"url":"https://github.com/kstrat2001/idor-check-eval/blob/e3e5291d383e0f9f5f6ec5cb35ce1a5765b07d3f/eval/hidden/test_hidden_sharing.py#L37"}],"kind":"coding","language":"python"},{"slug":"webhook-forgery-eval-1-3","title":"webhook-forgery-eval 1.3","evalSlug":"webhook-forgery-eval","evalName":"webhook-forgery-eval","version":"1.3","commitDate":"2026-10-28T06:52:06.000Z","current":true,"versions":[{"slug":"webhook-forgery-eval-1-3","title":"webhook-forgery-eval 1.3","version":"1.3","commitSha":"ddc2805d1967947b6202c02cd730867bd1057472","commitDate":"2026-10-28T06:52:06.000Z","approvedAt":"2026-10-28T08:44:53.562Z","current":true},{"slug":"webhook-forgery-eval-1-2","title":"webhook-forgery-eval 1.2","version":"1.2","commitSha":"6e6a6a07b3213b09dc59dcdfb79351303008a588","commitDate":"2026-10-27T13:41:04.000Z","approvedAt":"2026-10-27T14:10:34.964Z","current":false},{"slug":"webhook-forgery-eval-1-1","title":"webhook-forgery-eval 1.1","version":"1.1","commitSha":"925b530b5b22999daf792ec6ae3f1d698d5c662a","commitDate":null,"approvedAt":"2026-10-25T15:24:18.251Z","current":false}],"repo":"kstrat2001/webhook-forgery-eval","commitSha":"ddc2805d1967947b6202c02cd730867bd1057472","manifestHash":"80c0de52a79160fa03c45e269e38902109e3f0a574d6b48ad1aed72d4f688c41","approvedAt":"2026-10-28T08:44:53.562Z","demo":false,"provisionalNote":null,"summary":"Payment providers, code hosts and messaging services tell an app that something happened by sending an HTTP POST to a public URL: a webhook. Anyone who learns that URL can send a POST too. If the…","runs":12,"referenceRunId":"bcKBc2kImsqB","criteria":[{"id":"secret-required","name":"Secret required","description":"The receiver refuses to start without a non-empty signing secret.","path":"src/webhooks.js","lineStart":27,"lineEnd":27,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/webhooks.js#L27"},{"id":"missing-signature","name":"Missing signature","description":"A delivery with no Webhook-Signature header is rejected.","path":"src/signature.js","lineStart":68,"lineEnd":68,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/signature.js#L68"},{"id":"timestamp-tolerance","name":"Timestamp tolerance","description":"A delivery more than 5 minutes from the server clock, in either direction, is rejected.","path":"src/signature.js","lineStart":75,"lineEnd":75,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/signature.js#L75"},{"id":"raw-body","name":"Raw body","description":"The signature is checked over the raw request body exactly as received.","path":"src/signature.js","lineStart":79,"lineEnd":79,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/signature.js#L79"},{"id":"secret-rotation","name":"Secret rotation","description":"A delivery signed with the new or the old secret is accepted, and any of several v1 signatures may match.","path":"src/signature.js","lineStart":78,"lineEnd":78,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/signature.js#L78"},{"id":"applied-once","name":"Applied once","description":"A redelivered event is acknowledged but not applied to the ledger again.","path":"src/webhooks.js","lineStart":77,"lineEnd":77,"url":"https://github.com/kstrat2001/webhook-forgery-eval/blob/ddc2805d1967947b6202c02cd730867bd1057472/src/webhooks.js#L77"}],"kind":"coding","language":"nodejs"},{"slug":"salt-shaker-1-3","title":"salt-shaker 1.3","evalSlug":"salt-shaker","evalName":"salt-shaker","version":"1.3","commitDate":"2026-10-28T06:51:58.000Z","current":true,"versions":[{"slug":"salt-shaker-1-3","title":"salt-shaker 1.3","version":"1.3","commitSha":"71acd9a27d3a0c26fb33946f0cd55b8d85e190c3","commitDate":"2026-10-28T06:51:58.000Z","approvedAt":"2026-10-28T08:44:51.075Z","current":true},{"slug":"salt-shaker-1-2","title":"salt-shaker 1.2","version":"1.2","commitSha":"843c9b754ac58b4db34dd5a61da3d9bdc7de4f3f","commitDate":"2026-10-27T13:43:45.000Z","approvedAt":"2026-10-27T14:33:55.688Z","current":false},{"slug":"salt-shaker-1-1","title":"salt-shaker 1.1","version":"1.1","commitSha":"645767d508a1b643bbfa3dec97105213f71a5bfe","commitDate":null,"approvedAt":"2026-10-25T15:24:24.438Z","current":false}],"repo":"kstrat2001/salt-shaker","commitSha":"71acd9a27d3a0c26fb33946f0cd55b8d85e190c3","manifestHash":"1b41e669220162aba32affdd9a657a4887e9a459bdb94fda6d456e78b58cc89e","approvedAt":"2026-10-28T08:44:51.075Z","demo":false,"provisionalNote":null,"summary":"Every service that stores passwords will one day face the question of what happens if its user table leaks. With a fast unsalted hash like SHA-1 or SHA-256, an attacker with a GPU recovers most…","runs":12,"referenceRunId":"aK6GZZRtTWEI","criteria":[{"id":"hash-format","name":"Salted scrypt hash","description":"hashPassword returns scrypt$N$r$p$salt$hash at N=16384, r=8, p=1, with a fresh 16-byte salt every call.","path":"eval/hidden/hidden.test.js","lineStart":30,"lineEnd":30,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L30"},{"id":"verify-never-throws","name":"Verify never throws","description":"verifyPassword accepts the right password, refuses a wrong one, and returns false for a malformed stored value instead of throwing.","path":"eval/hidden/hidden.test.js","lineStart":51,"lineEnd":51,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L51"},{"id":"stored-parameters","name":"Parameters read from the hash","description":"verifyPassword reads N, r and p from the stored string, so a hash made with other parameters still verifies.","path":"eval/hidden/hidden.test.js","lineStart":63,"lineEnd":63,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L63"},{"id":"nfc-normalized","name":"Unicode normalized","description":"Composed and decomposed forms of the same password both verify.","path":"eval/hidden/hidden.test.js","lineStart":67,"lineEnd":67,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L67"},{"id":"weak-password","name":"Weak passwords refused","description":"register and changePassword throw WeakPassword below 12 or above 256 characters, counted after normalization, and store nothing.","path":"eval/hidden/hidden.test.js","lineStart":75,"lineEnd":75,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L75"},{"id":"same-error","name":"No account enumeration","description":"An unknown email and a wrong password throw the same InvalidCredentials, and a wrong current password changes nothing.","path":"eval/hidden/hidden.test.js","lineStart":100,"lineEnd":100,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L100"},{"id":"legacy-upgrade","name":"Legacy hashes upgraded","description":"A successful login upgrades a sha256 hash to scrypt; a failed login leaves it untouched.","path":"eval/hidden/hidden.test.js","lineStart":118,"lineEnd":118,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L118"},{"id":"weak-rehash","name":"Weaker scrypt rehashed","description":"A successful login with a scrypt hash below N=16384 rehashes it at the current parameters.","path":"eval/hidden/hidden.test.js","lineStart":137,"lineEnd":137,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L137"},{"id":"no-plaintext","name":"No plaintext stored","description":"The stored record never contains the plaintext password, in any field.","path":"eval/hidden/hidden.test.js","lineStart":144,"lineEnd":144,"url":"https://github.com/kstrat2001/salt-shaker/blob/71acd9a27d3a0c26fb33946f0cd55b8d85e190c3/eval/hidden/hidden.test.js#L144"}],"kind":"coding","language":"typescript"},{"slug":"pepper-grinder-2-3","title":"pepper-grinder 2.3","evalSlug":"pepper-grinder","evalName":"pepper-grinder","version":"2.3","commitDate":"2026-10-28T06:51:50.000Z","current":true,"versions":[{"slug":"pepper-grinder-2-3","title":"pepper-grinder 2.3","version":"2.3","commitSha":"ffb6216e4af8d3a8a38cdf3f96322f43b3185d45","commitDate":"2026-10-28T06:51:50.000Z","approvedAt":"2026-10-28T08:44:48.975Z","current":true},{"slug":"pepper-grinder-2-2","title":"pepper-grinder 2.2","version":"2.2","commitSha":"cf60deca214989868000d38b37546b801c1cc0dd","commitDate":"2026-10-27T13:43:01.000Z","approvedAt":"2026-10-27T14:33:48.063Z","current":false},{"slug":"pepper-grinder-2-1","title":"pepper-grinder 2.1","version":"2.1","commitSha":"1db2958e69051d914deb4b251535e4285ae8e713","commitDate":null,"approvedAt":"2026-10-24T03:53:42.635Z","current":false}],"repo":"kstrat2001/pepper-grinder","commitSha":"ffb6216e4af8d3a8a38cdf3f96322f43b3185d45","manifestHash":"ac3d035d260af8069fc757167e7ba008506e41adeaae80ef848ccd784b5d2c0c","approvedAt":"2026-10-28T08:44:48.975Z","demo":false,"provisionalNote":null,"summary":null,"runs":5,"referenceRunId":"yUgHjVwbCzcu","criteria":[{"id":"pepper","name":"Pepper in the token hash","description":"Tokens are hashed at rest with the configured pepper, so a test fails if the pepper is dropped from the hash.","path":"src/tokenRotation.js","lineStart":53,"lineEnd":53,"url":"https://github.com/kstrat2001/pepper-grinder/blob/ffb6216e4af8d3a8a38cdf3f96322f43b3185d45/src/tokenRotation.js#L53"},{"id":"row-lock","name":"Row lock during rotation","description":"Two concurrent rotations of the same token cannot both succeed, so a test fails if rotation stops taking the per-row lock.","path":"src/tokenRotation.js","lineStart":101,"lineEnd":101,"url":"https://github.com/kstrat2001/pepper-grinder/blob/ffb6216e4af8d3a8a38cdf3f96322f43b3185d45/src/tokenRotation.js#L101"},{"id":"boot-guard","name":"Boot guard on the dev pepper","description":"Outside development the service refuses to start with the dev placeholder pepper, and in development it starts.","path":"src/tokenRotation.js","lineStart":33,"lineEnd":33,"url":"https://github.com/kstrat2001/pepper-grinder/blob/ffb6216e4af8d3a8a38cdf3f96322f43b3185d45/src/tokenRotation.js#L33"},{"id":"expiry-parsing","name":"Expiry parsing","description":"An expired or unparseable expiry timestamp is treated as expired, whether stored as a Date or an ISO string.","path":"src/tokenRotation.js","lineStart":130,"lineEnd":130,"url":"https://github.com/kstrat2001/pepper-grinder/blob/ffb6216e4af8d3a8a38cdf3f96322f43b3185d45/src/tokenRotation.js#L130"}],"kind":"coding","language":"nodejs"}],"meta":{"timestamp":"2026-11-18T23:35:10.355Z","count":7}}