From d636eed20b349c463d8fb05cfd062fa0192b19fe Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:33:44 -0800 Subject: [PATCH] fix(object-hasher): switch to object-hash to fix hashing of large objects (#7591) --- .changeset/breezy-poems-act.md | 5 +++++ .changeset/strange-ghosts-visit.md | 5 +++++ crypto/object-hasher/package.json | 5 +++-- crypto/object-hasher/src/index.ts | 36 +++++++++++++++++++++++++++--- crypto/object-hasher/test/index.ts | 16 +++++++++++-- pnpm-lock.yaml | 21 ++++++++++++----- 6 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 .changeset/breezy-poems-act.md create mode 100644 .changeset/strange-ghosts-visit.md diff --git a/.changeset/breezy-poems-act.md b/.changeset/breezy-poems-act.md new file mode 100644 index 0000000000..c045ab3113 --- /dev/null +++ b/.changeset/breezy-poems-act.md @@ -0,0 +1,5 @@ +--- +"pnpm": patch +--- + +Use the `object-hash` library instead of `node-object-hash` for hashing keys of side-effects cache [#7591](https://github.com/pnpm/pnpm/pull/7591). diff --git a/.changeset/strange-ghosts-visit.md b/.changeset/strange-ghosts-visit.md new file mode 100644 index 0000000000..0df1bdce45 --- /dev/null +++ b/.changeset/strange-ghosts-visit.md @@ -0,0 +1,5 @@ +--- +"@pnpm/crypto.object-hasher": major +--- + +fix(object-hasher): switch to object-hash to fix hashing of large objects diff --git a/crypto/object-hasher/package.json b/crypto/object-hasher/package.json index f9ba4c89ec..fd23b53076 100644 --- a/crypto/object-hasher/package.json +++ b/crypto/object-hasher/package.json @@ -30,10 +30,11 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/crypto/object-hasher#readme", "dependencies": { - "node-object-hash": "3.0.0" + "object-hash": "3.0.0" }, "devDependencies": { - "@pnpm/crypto.object-hasher": "workspace:*" + "@pnpm/crypto.object-hasher": "workspace:*", + "@types/object-hash": "3.0.6" }, "funding": "https://opencollective.com/pnpm", "exports": { diff --git a/crypto/object-hasher/src/index.ts b/crypto/object-hasher/src/index.ts index 5724fa6d7e..7f4e49478a 100644 --- a/crypto/object-hasher/src/index.ts +++ b/crypto/object-hasher/src/index.ts @@ -1,4 +1,34 @@ -import { hasher } from 'node-object-hash' +// We use object-hash even though node-object-hash is faster. +// Unlike node-object-hash, object-hash is streaming the hash updates, +// avoiding "Invalid string length" errors. +import hash from 'object-hash' -export const hashObjectWithoutSorting = hasher({ sort: false }).hash -export const hashObject = hasher({}).hash +const defaultOptions: hash.BaseOptions = { + respectType: false, + algorithm: 'sha1', +} + +const withoutSortingOptions: hash.BaseOptions = { + ...defaultOptions, + unorderedArrays: false, + unorderedObjects: false, + unorderedSets: false, +} + +const withSortingOptions: hash.BaseOptions = { + ...defaultOptions, + unorderedArrays: true, + unorderedObjects: true, + unorderedSets: true, +} + +function hashUnknown (object: unknown, options: hash.BaseOptions) { + if (object === undefined) { + // '0'.repeat(40) to match the length of other returned sha1 hashes. + return '0000000000000000000000000000000000000000' + } + return hash(object, options) +} + +export const hashObjectWithoutSorting = (object: unknown) => hashUnknown(object, withoutSortingOptions) +export const hashObject = (object: unknown) => hashUnknown(object, withSortingOptions) diff --git a/crypto/object-hasher/test/index.ts b/crypto/object-hasher/test/index.ts index 7e4396186d..40dd99bb41 100644 --- a/crypto/object-hasher/test/index.ts +++ b/crypto/object-hasher/test/index.ts @@ -1,13 +1,25 @@ import { hashObject, hashObjectWithoutSorting } from '@pnpm/crypto.object-hasher' describe('hashObject', () => { + const hash = hashObject it('creates a hash', () => { - expect(hashObject({ b: 1, a: 2 })).toEqual('c8c943f9321eb7f98834b58391eee848d458c7b35211fc4911cdb1bbd877b74a') + expect(hash({ b: 1, a: 2 })).toEqual('e3d3f89836fac144779e57d0e831efd06336036b') + expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000') + }) + it('sorts', () => { + expect(hash({ b: 1, a: 2 })).toEqual(hash({ a: 2, b: 1 })) + expect(hash({ b: new Set([1, 2, 3]), a: [1, 2, 3] })).toEqual(hash({ a: [2, 3, 1], b: new Set([3, 2, 1]) })) }) }) describe('hashObjectWithoutSorting', () => { + const hash = hashObjectWithoutSorting it('creates a hash', () => { - expect(hashObjectWithoutSorting({ b: 1, a: 2 })).toEqual('c0a68b14aa1886a799c2e7c1289b65ccb79a668881d5b6956f0b185a9ac112d7') + expect(hash({ b: 1, a: 2 })).toEqual('dd34c1644a1d52da41808e5c1e6849829ef77999') + expect(hash(undefined)).toEqual('0000000000000000000000000000000000000000') + }) + it('does not sort', () => { + expect(hash({ b: 1, a: 2 })).not.toEqual(hash({ a: 2, b: 1 })) + expect(hash({ b: new Set([1, 2, 3]), a: [1, 2, 3] })).not.toEqual(hash({ a: [2, 3, 1], b: new Set([3, 2, 1]) })) }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7dabc76ba2..e968dcc2f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -776,13 +776,16 @@ importers: crypto/object-hasher: dependencies: - node-object-hash: + object-hash: specifier: 3.0.0 version: 3.0.0 devDependencies: '@pnpm/crypto.object-hasher': specifier: workspace:* version: 'link:' + '@types/object-hash': + specifier: 3.0.6 + version: 3.0.6 dedupe/check: dependencies: @@ -9131,6 +9134,10 @@ packages: /@types/normalize-path@3.0.2: resolution: {integrity: sha512-DO++toKYPaFn0Z8hQ7Tx+3iT9t77IJo/nDiqTXilgEP+kPNIYdpS9kh3fXuc53ugqwp9pxC1PVjCpV1tQDyqMA==} + /@types/object-hash@3.0.6: + resolution: {integrity: sha512-fOBV8C1FIu2ELinoILQ+ApxcUKz4ngq+IWUYrxSGjXzzjUALijilampwkMgEtJ+h2njAW3pi853QpzNVCHB73w==} + dev: true + /@types/parse-json@4.0.2: resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} dev: true @@ -14916,11 +14923,6 @@ packages: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true - /node-object-hash@3.0.0: - resolution: {integrity: sha512-jLF6tlyletktvSAawuPmH1SReP0YfZQ+tBrDiTCK+Ai7eXPMS9odi5xW/iKC7ZhrWJJ0Z5xYcW/x+1fVMn1Qvw==} - engines: {node: '>=16', pnpm: '>=8'} - dev: false - /node-releases@2.0.14: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true @@ -15078,6 +15080,11 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + /object-hash@3.0.0: + resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} + engines: {node: '>= 6'} + dev: false + /object-inspect@1.13.1: resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} @@ -18154,5 +18161,7 @@ packages: dev: false time: + /@types/object-hash@3.0.6: '2023-11-07T12:14:41.852Z' /fuse-native@2.2.6: '2020-06-03T19:26:36.838Z' /node-gyp@10.0.1: '2023-11-02T18:13:42.360Z' + /object-hash@3.0.0: '2022-02-18T15:10:47.040Z'