diff --git a/graphile/graphile-presigned-url-plugin/README.md b/graphile/graphile-presigned-url-plugin/README.md index c67d3c3355..e0cfacb59a 100644 --- a/graphile/graphile-presigned-url-plugin/README.md +++ b/graphile/graphile-presigned-url-plugin/README.md @@ -19,7 +19,7 @@ Presigned URL upload plugin for PostGraphile v5. - `requestUploadUrl` mutation — generates presigned PUT URLs for direct client-to-S3 upload - `downloadUrl` computed field — presigned GET URLs for private files, public URLs for public files - `maxBulkFiles` field on bucket types — the most files one bulk-upload call accepts (`storage_module.max_bulk_files`, default 100) -- Content-hash based S3 keys (SHA-256) with automatic deduplication — only against rows whose bytes are in S3, so a retry of a never-completed upload gets a fresh URL +- Independent server-allocated object keys (`originals/`), with SHA-256 integrity and deduplication of confirmed content in the same logical slot - Per-bucket MIME type and file size validation ## Usage @@ -44,3 +44,23 @@ const preset = { ], }; ``` + +## Upload identity and retries + +The optional upload input `key` names a **logical slot** (`files.logical_key`); +without it the logical slot is the SHA-256 content hash. It never chooses an S3 +location. The generated recorder allocates a fresh `originals/` for each +new request and returns it as `files.key`. The response `key`, projection `key`, +PUT signature and downloads all use that physical key. + +A confirmed (`uploaded`/`processed`) row with the same bucket, logical key and +checksum is reused, without issuing a PUT URL. Pending or rejected attempts do +not deduplicate: a new request gets a new object and confirm job, and leaves the +older attempt to its existing lifecycle. Repeating a PUT with the same URL is a +retry of that same object; checksum and upload-id must still match its signature. + +This requires the matching constructive-db storage generator with `logical_key` +and a server-allocated recorder result. Deploy the generator and pinned package +together; there is no fallback to signing the caller's logical key. Successful +logical replacement retains the existing database authorization policy; UUIDs +are object identities, not access control. diff --git a/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts index 1d37788372..ebc02f6b41 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/custom-key-upload.test.ts @@ -10,6 +10,7 @@ */ import { processSingleFile } from '../src/plugin'; +import { generatePresignedPutUrl } from '../src/s3-signer'; import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types'; jest.mock('../src/s3-signer', () => ({ @@ -21,6 +22,8 @@ const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; const OLD_FILE_ID = '44444444-4444-4444-4444-444444444444'; const NEW_FILE_ID = '55555555-5555-5555-5555-555555555555'; const KEY = 'index.html'; +const OLD_KEY = `originals/${OLD_FILE_ID}`; +const NEW_KEY = `originals/${NEW_FILE_ID}`; function storageConfig(overrides: Partial = {}): StorageModuleConfig { return { @@ -60,15 +63,15 @@ function fakeTx(existingHash: string, deletable = true, existingStatus = 'upload const txClient = { async query(opts: { text: string; values: unknown[] }) { queries.push(opts); - if (/^SELECT id, status/.test(opts.text)) { + if (/^SELECT id, (?:key|status)/.test(opts.text) && !/record_file\(/.test(opts.text)) { // The probe is by (key, bucket, hash): a row carrying other bytes is not seen. const probedHash = opts.values[2]; - return { rows: probedHash === existingHash ? [{ id: OLD_FILE_ID, status: existingStatus }] : [] }; + return { rows: probedHash === existingHash && ['uploaded', 'processed'].includes(existingStatus) ? [{ id: OLD_FILE_ID, key: OLD_KEY, status: existingStatus }] : [] }; } if (/^DELETE FROM storage_public\.app_files/.test(opts.text)) { return { rows: deletable ? [{ id: OLD_FILE_ID }] : [] }; } - if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID }] }; + if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID, key: NEW_KEY }] }; throw new Error(`unexpected query: ${opts.text}`); }, }; @@ -96,7 +99,7 @@ describe('custom-key upload of changed bytes', () => { expect(result).toMatchObject({ fileId: NEW_FILE_ID, - key: KEY, + key: NEW_KEY, deduplicated: false, uploadUrl: 'https://s3.example.com/put', }); @@ -116,23 +119,29 @@ describe('custom-key upload of changed bytes', () => { const result = await upload(storageConfig(), txClient); expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false }); - expect(queries.map((q) => q.text.trim().split(/\s+/)[0])).toEqual(['SELECT', 'DELETE', 'SELECT']); - expect(queries[1].values).toEqual([OLD_FILE_ID]); + expect(queries.map((q) => q.text.trim().split(/\s+/)[0])).toEqual(['SELECT', 'SELECT']); + expect(result.key).toBe(NEW_KEY); + expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false); }); - it('refuses to restart a row the caller cannot delete', async () => { + it('retries independently even when the caller cannot delete an earlier attempt', async () => { const { txClient, queries } = fakeTx('b'.repeat(64), false, 'requested'); - - const refused = upload(storageConfig(), txClient); - await expect(refused).rejects.toThrow( - `FILE_NOT_REPLACEABLE: file ${OLD_FILE_ID} at key ${KEY}`, - ); - // A named error carries its code, so the server passes it through - // instead of masking it as INTERNAL_SERVER_ERROR. - await expect(refused).rejects.toMatchObject({ code: 'FILE_NOT_REPLACEABLE', errorClass: 'public' }); - expect(queries.some((q) => /record_file\(/.test(q.text))).toBe(false); + await expect(upload(storageConfig(), txClient)).resolves.toMatchObject({ key: NEW_KEY, deduplicated: false }); + expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false); }); + it.each(['a'.repeat(64), OLD_KEY, `derived/${OLD_KEY}/thumb.webp`])( + 'never signs the attacker-selected logical key %s', async (logicalKey) => { + const { txClient } = fakeTx('a'.repeat(64)); + const result = await processSingleFile(options, txClient, storageConfig(), 'db', bucket, s3, { + contentHash: 'b'.repeat(64), contentType: 'text/html', size: 16, key: logicalKey, + }); + expect(result.key).toBe(NEW_KEY); + expect(result.file.key).toBe(NEW_KEY); + expect(generatePresignedPutUrl).toHaveBeenLastCalledWith(s3, NEW_KEY, 'text/html', 16, 900, 'b'.repeat(64), NEW_FILE_ID); + }, + ); + it('deduplicates identical bytes at the key without touching the row', async () => { const { txClient, queries } = fakeTx('b'.repeat(64)); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts index f1bbbc56ed..3ecdebab43 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts @@ -70,8 +70,8 @@ describe('validateCustomKey', () => { expect(validateCustomKey(key)).toMatch(/^INVALID_KEY:/); }); - // A file's rendered versions live under `.versions/` and are served to - // whoever can read that file, so no file of its own may be stored there. + // The legacy `.versions/` suffix remains reserved in logical-key syntax; + // physical storage isolation no longer depends on this validation. it.each(['photo.png.versions/secret.webp', 'a/b.versions/c/d.txt'])('rejects %s: the versions prefix is reserved', (key) => { expect(validateCustomKey(key)).toMatch(/reserved/); }); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts index 061a694f50..40b147dc8f 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts @@ -86,9 +86,9 @@ function fakeTx(rowKey: string, status: string) { const txClient = { async query(opts: { text: string; values: unknown[] }) { queries.push(opts); - if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID }] }; + if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID, key: `originals/${NEW_FILE_ID}` }] }; if (/^SELECT id/.test(opts.text)) { - return { rows: opts.values[0] === rowKey ? [{ id: OLD_FILE_ID, status }] : [] }; + return { rows: opts.values[0] === rowKey && ['uploaded', 'processed'].includes(status) ? [{ id: OLD_FILE_ID, key: `originals/${OLD_FILE_ID}` }] : [] }; } if (/^DELETE FROM storage_public\.app_files/.test(opts.text)) return { rows: [{ id: OLD_FILE_ID }] }; throw new Error(`unexpected query: ${opts.text}`); @@ -125,7 +125,7 @@ describe.each([ }); expect(heads(s3)).toEqual([]); const del = queries.find((q) => /^DELETE/.test(q.text)); - expect(del?.values).toEqual([OLD_FILE_ID]); + expect(del).toBeUndefined(); }, ); @@ -152,8 +152,8 @@ describe('content-addressed upload whose hash a custom-key row carries', () => { const result = await upload(txClient, s3); - expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false, key: HASH }); - expect(queries[0].values).toEqual([HASH, BUCKET_ID]); + expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false, key: `originals/${NEW_FILE_ID}` }); + expect(queries[0].values).toEqual([HASH, BUCKET_ID, HASH]); expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false); }); }); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts index 54206da86a..de9e43a6c2 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/storage-file-recorder.test.ts @@ -50,16 +50,14 @@ function input() { describe('recordManagedFile', () => { it('calls the generated recorder with only base object facts', async () => { const query = jest.fn().mockResolvedValue({ - rows: [{ id: '33333333-3333-3333-3333-333333333333' }], + rows: [{ id: '33333333-3333-3333-3333-333333333333', key: 'originals/server-generated' }], }); - await expect(recordManagedFile({ query }, storageConfig(), input())).resolves.toBe( - '33333333-3333-3333-3333-333333333333', - ); + await expect(recordManagedFile({ query }, storageConfig(), input())).resolves.toEqual({ id: '33333333-3333-3333-3333-333333333333', key: 'originals/server-generated' }); expect(query).toHaveBeenCalledWith({ text: expect.stringContaining( - 'SELECT id FROM storage_private.app_files_record_file(bucket_id := $1::uuid, key := $2::text', + 'SELECT id, key FROM storage_private.app_files_record_file(bucket_id := $1::uuid, key := $2::text', ), values: [ input().bucketId, @@ -76,7 +74,7 @@ describe('recordManagedFile', () => { }); it('appends a supported path argument by name and never a version link', async () => { - const query = jest.fn().mockResolvedValue({ rows: [{ id: 'file-id' }] }); + const query = jest.fn().mockResolvedValue({ rows: [{ id: 'file-id', key: 'originals/server-generated' }] }); await recordManagedFile( { query }, @@ -103,6 +101,20 @@ describe('recordManagedFile', () => { }); }); + it.each([ + undefined, + { id: 'file-id' }, + { id: 'file-id', key: '' }, + { key: 'originals/server-generated' }, + { id: 'file-id', key: input().key }, + ])('refuses an invalid or echoed recorder identity: %p', async (row) => { + const query = jest.fn().mockResolvedValue({ rows: row ? [row] : [] }); + await expect(recordManagedFile({ query }, storageConfig(), input())).rejects.toMatchObject({ + code: 'INTERNAL_ERROR', + errorClass: 'internal', + }); + }); + it('fails loudly when a path is requested on a module without path shares', async () => { const query = jest.fn(); diff --git a/graphile/graphile-presigned-url-plugin/src/custom-key.ts b/graphile/graphile-presigned-url-plugin/src/custom-key.ts index ec3bbfe1f7..efc1c36d9a 100644 --- a/graphile/graphile-presigned-url-plugin/src/custom-key.ts +++ b/graphile/graphile-presigned-url-plugin/src/custom-key.ts @@ -38,8 +38,7 @@ export function validateCustomKey(key: string): string | null { if (key.includes('\0')) { return 'INVALID_KEY: null bytes not allowed'; } - // `.versions/` is where a file's rendered versions are stored, and they - // are served to whoever can read that file: no file of its own may live there. + // Preserve the existing logical-key syntax restriction on the legacy suffix. if (key.includes('.versions/')) { return 'INVALID_KEY: ".versions/" is reserved for a file\'s rendered versions'; } diff --git a/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts b/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts index 6b2dbd4c05..fd22c892fc 100644 --- a/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts +++ b/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts @@ -5,7 +5,7 @@ * (`FILE_REF_DOMAINS`: `image`, `upload`), or an array of one. It holds a * document that projects a files row — `id`, `key`, `bucket_id` — and, when a * lane has rendered versions beside the source, lists them under `versions` - * (`.versions/-.`). The column reaches GraphQL as JSON, + * (`derived//-.`). The column reaches GraphQL as JSON, * so without these fields a client has no URL to fetch the file with. * * For each such column `photo` on a table type this adds @@ -23,7 +23,7 @@ * 2. the source key, bucket, filename and publicness come from that row, * never from the document; * 3. a version is signed only when its key is a direct child of that row's - * own `.versions/` prefix. A key the document merely names is never + * own `derived//` prefix. A key the document merely names is never * signed. * * Files rows are read once per request, not once per document: every document @@ -88,11 +88,11 @@ type PgClientLike = { /** * Whether `variantKey` is an object a lane wrote beside the file at - * `sourceKey`: a direct child of `.versions/`. + * `sourceKey`: a direct child of `derived//`. */ export function isVersionKeyOf(sourceKey: string, variantKey: unknown): variantKey is string { if (typeof variantKey !== 'string' || !sourceKey) return false; - const prefix = `${sourceKey}.versions/`; + const prefix = `derived/${sourceKey}/`; if (!variantKey.startsWith(prefix)) return false; const name = variantKey.slice(prefix.length); return name.length > 0 && !name.includes('/') && name !== '.' && name !== '..'; diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 752a4f2218..c7af9099cd 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -27,8 +27,6 @@ import { checkTypeAgreement } from 'mime-bytes'; import { validateCustomKey } from './custom-key'; import { resolveDefaultBucket } from './default-bucket'; -import { isLiveFileRow } from './file-lifecycle'; -import { fileNotReplaceable } from './file-not-replaceable'; import { buildFileProjection, type FileProjection } from './file-projection'; import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket'; import { generatePresignedPutUrl, presignedPutHeaders } from './s3-signer'; @@ -50,10 +48,6 @@ function isValidSha256(hash: string): boolean { return SHA256_HEX_REGEX.test(hash); } -function buildS3Key(contentHash: string): string { - return contentHash; -} - function derivePathFromKey(key: string): string | null { const lastSlash = key.lastIndexOf('/'); if (lastSlash <= 0) return null; @@ -201,7 +195,7 @@ export function createPresignedUrlPlugin( contentType: { type: new GraphQLNonNull(GraphQLString), description: 'MIME type of the file' }, size: { type: new GraphQLNonNull(GraphQLInt), description: 'File size in bytes' }, filename: { type: GraphQLString, description: 'Original filename (optional)' }, - key: { type: GraphQLString, description: 'Custom S3 key (only when bucket has allow_custom_keys=true)' }, + key: { type: GraphQLString, description: 'Custom logical key (only when bucket has allow_custom_keys=true)' }, }, }); @@ -337,7 +331,7 @@ export function createPresignedUrlPlugin( contentType: { type: new GraphQLNonNull(GraphQLString), description: 'MIME type of the file' }, size: { type: new GraphQLNonNull(GraphQLInt), description: 'File size in bytes' }, filename: { type: GraphQLString, description: 'Original filename (optional)' }, - key: { type: GraphQLString, description: 'Custom S3 key (only when bucket has allow_custom_keys=true)' }, + key: { type: GraphQLString, description: 'Custom logical key (only when bucket has allow_custom_keys=true)' }, }, }); @@ -564,135 +558,57 @@ export async function processSingleFile( s3ForDb, ); - // Determine S3 key - let s3Key: string; - let isCustomKey = false; + // The user names a logical slot, never an object-store location. Only the + // recorder allocates the physical key, independently for every new request. + const logicalKey = customKey || contentHash; if (customKey) { if (!bucket.allow_custom_keys) { throw new Error('CUSTOM_KEY_NOT_ALLOWED: bucket does not allow custom keys'); } const keyError = validateCustomKey(customKey); - if (keyError) { - throw new Error(keyError); - } - s3Key = customKey; - isCustomKey = true; - } else { - s3Key = buildS3Key(contentHash); + if (keyError) throw new Error(keyError); } - // Dedup check. A key is one stored object: a row at the key with the same - // hash already describes these bytes, and a live one is a dedup hit. - // - // A row whose bytes never landed must not be reported as a dedup hit: the - // caller would store a reference to an object that is not in S3. Such a row is - // dropped instead, and the upload proceeds as a fresh one below — the insert is - // what enqueues the confirm-upload job, so restarting the lifecycle is the only - // way the retry can ever leave `requested`. Dropping it is also what keeps the - // retry insertable at all for a content-addressed key, where the key *is* the - // hash and a second row would collide on (bucket_id, key). The GC job the - // delete enqueues re-takes the reference count when it runs (≥5s later), by - // which point the replacement row exists, so it no-ops. - // - // A custom key holding different bytes is replaced in place, by the database: - // the new row is inserted as `requested` beside the current one (its BEFORE - // INSERT trigger refuses a key held by a row this caller cannot read), the PUT - // through its URL overwrites the object, and its confirm deletes the row it - // replaced. Until then the current row stays listable and downloadable, so - // nothing is deleted here and no version link is made — a custom key has no - // version chain. - let staleFileId: string | null = null; - - if (isCustomKey) { - const existingResult = await txClient.query({ - text: `SELECT id, status - FROM ${storageConfig.filesQualifiedName} - WHERE key = $1 - AND bucket_id = $2 - AND content_hash = $3 - ORDER BY created_at DESC - LIMIT 1`, - values: [s3Key, bucket.id, contentHash], - }); - - if (existingResult.rows.length > 0) { - const existing = existingResult.rows[0]; - if (isLiveFileRow(existing)) { - log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`); - return { - uploadUrl: null as string | null, - uploadHeaders: null as Record | null, - fileId: existing.id as string, - key: s3Key, - deduplicated: true, - expiresAt: null as string | null, - file: projectFile(existing.id as string, s3Key), - }; - } - staleFileId = existing.id as string; - log.info(`Restarting upload of key ${s3Key}: file ${staleFileId} has no bytes behind it`); - } - } else { - // Only the row at the hash's own key names these bytes for good: a custom-key - // row carrying the hash holds them at a key whose next upload replaces them. - const dedupResult = await txClient.query({ - text: `SELECT id, status + // Dedup is a read of confirmed identical content in the same logical slot. + // It issues no PUT URL. Pending/rejected attempts are not reused or deleted: + // each retry gets its own object and lifecycle, including its expiry/GC. + const existingResult = await txClient.query({ + text: `SELECT id, key FROM ${storageConfig.filesQualifiedName} - WHERE key = $1 - AND bucket_id = $2 + WHERE logical_key = $1 AND bucket_id = $2 AND content_hash = $3 + AND status IN ('uploaded', 'processed') LIMIT 1`, - values: [s3Key, bucket.id], - }); - - if (dedupResult.rows.length > 0) { - const existingFile = dedupResult.rows[0]; - if (isLiveFileRow(existingFile)) { - log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); - - return { - uploadUrl: null as string | null, - uploadHeaders: null as Record | null, - fileId: existingFile.id as string, - key: s3Key, - deduplicated: true, - expiresAt: null as string | null, - file: projectFile(existingFile.id as string, s3Key), - }; - } - staleFileId = existingFile.id as string; - log.info(`Restarting upload of hash ${contentHash}: file ${staleFileId} has no bytes behind it`); - } - } - - if (staleFileId !== null) { - // The DELETE runs under the request role, so RLS decides whether this caller - // may drop the row. A row the caller can see but not delete stays, and the - // upload fails here rather than colliding on (bucket_id, key) below. - const deleted = await txClient.query({ - text: `DELETE FROM ${storageConfig.filesQualifiedName} WHERE id = $1 RETURNING id`, - values: [staleFileId], - }); - if (deleted.rows.length === 0) { - throw fileNotReplaceable(staleFileId, s3Key); - } + values: [logicalKey, bucket.id, contentHash], + }); + if (existingResult.rows.length > 0) { + const existing = existingResult.rows[0]; + return { + uploadUrl: null as string | null, + uploadHeaders: null as Record | null, + fileId: existing.id as string, + key: existing.key as string, + deduplicated: true, + expiresAt: null as string | null, + file: projectFile(existing.id as string, existing.key as string), + }; } - // Auto-derive ltree path from custom key directory (only when has_path_shares) - const derivedPath = isCustomKey && storageConfig.hasPathShares ? derivePathFromKey(s3Key) : null; - - const fileId = await recordManagedFile(txClient, storageConfig, { + const derivedPath = customKey && storageConfig.hasPathShares ? derivePathFromKey(logicalKey) : null; + const recorded = await recordManagedFile(txClient, storageConfig, { bucketId: bucket.id, - key: s3Key, + key: logicalKey, contentHash, mimeType: contentType, size, filename: filename || null, path: derivedPath, }); + const fileId = recorded.id; + const s3Key = recorded.key; // Generate presigned PUT URL, signed over the client's SHA-256 so the store - // only accepts bytes that hash to the content hash the row (and, for - // content-addressed buckets, the key and any later dedup hit) is named by. + // only accepts bytes that hash to the content hash on this row and any + // later dedup hit. Neither logical keys nor checksums locate the object. const uploadUrl = await generatePresignedPutUrl( s3ForDb, s3Key, diff --git a/graphile/graphile-presigned-url-plugin/src/storage-file-recorder.ts b/graphile/graphile-presigned-url-plugin/src/storage-file-recorder.ts index 86b7fe019f..44a0b4a7d3 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-file-recorder.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-file-recorder.ts @@ -1,7 +1,10 @@ +import { ConstructiveError } from '@constructive-io/errors'; + import type { StorageModuleConfig } from './types'; interface RecordManagedFileInput { bucketId: string; + /** Logical replacement/dedup slot; the recorder allocates the physical key. */ key: string; contentHash: string; mimeType: string; @@ -18,13 +21,14 @@ interface RecordManagedFileInput { * when the generated function supports them and the caller has a value. A * version's previous_version_id is not an argument: a version chain is written * only by a platform writer, and a custom key has none — a re-upload at a key - * replaces the row at it. + * replaces the row at it only after successful confirmation. The returned key + * is the server-allocated physical location, never input.key. */ export async function recordManagedFile( pgClient: { query: (opts: { text: string; values: unknown[] }) => Promise<{ rows: unknown[] }> }, storageConfig: StorageModuleConfig, input: RecordManagedFileInput, -): Promise { +): Promise<{ id: string; key: string }> { if (!storageConfig.recorderQualifiedName) { throw new Error( `STORAGE_RECORDER_MISSING: storage module ${storageConfig.id} (${storageConfig.filesTableName}) ` + @@ -63,8 +67,19 @@ export async function recordManagedFile( } const result = await pgClient.query({ - text: `SELECT id FROM ${storageConfig.recorderQualifiedName}(${args.join(', ')})`, + text: `SELECT id, key FROM ${storageConfig.recorderQualifiedName}(${args.join(', ')})`, values, }); - return (result.rows[0] as { id: string }).id; + const row = result.rows[0] as { id?: unknown; key?: unknown } | undefined; + if (!row || typeof row.id !== 'string' || !row.id || + typeof row.key !== 'string' || !row.key || row.key === input.key) { + throw new ConstructiveError({ + code: 'INTERNAL_ERROR', + message: 'Storage recorder did not allocate an independent object identity', + errorClass: 'internal', + http: 500, + context: { storageModuleId: storageConfig.id }, + }); + } + return { id: row.id, key: row.key }; } diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 8375d3d289..0021b8cfab 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -134,9 +134,9 @@ export interface RequestUploadUrlInput { /** Original filename (optional, for display/Content-Disposition) */ filename?: string; /** - * Custom S3 key for the file (only allowed when bucket has allow_custom_keys=true). + * Custom logical key for the file (only allowed when bucket has allow_custom_keys=true). * When omitted, key defaults to contentHash (content-addressed dedup). - * When provided, the file is stored at this key; dedup is bypassed. + * The server allocates an independent physical key; confirmed identical content in the logical slot is deduplicated. * Max 1024 chars. Must not contain path traversal (.. or leading /). */ key?: string; diff --git a/graphql/server-test/__fixtures__/seed/db-scope-storage/schema.sql b/graphql/server-test/__fixtures__/seed/db-scope-storage/schema.sql index aca3d4b149..82d90c4b21 100644 --- a/graphql/server-test/__fixtures__/seed/db-scope-storage/schema.sql +++ b/graphql/server-test/__fixtures__/seed/db-scope-storage/schema.sql @@ -33,7 +33,8 @@ COMMENT ON TABLE "tess-storage-public".buckets IS E'@storageBuckets\nStorage buc CREATE TABLE "tess-storage-public".files ( id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), bucket_id uuid NOT NULL REFERENCES "tess-storage-public".buckets(id), - key text NOT NULL, + key text NOT NULL DEFAULT 'originals/' || gen_random_uuid()::text, + logical_key text, content_hash text NOT NULL, mime_type text NOT NULL, size bigint, @@ -47,6 +48,9 @@ CREATE TABLE "tess-storage-public".files ( UNIQUE (bucket_id, key) ); +CREATE INDEX ON "tess-storage-public".files (bucket_id, logical_key, content_hash) + WHERE status IN ('uploaded', 'processed'); + COMMENT ON TABLE "tess-storage-public".files IS E'@storageFiles\nStorage files table'; GRANT SELECT, INSERT, UPDATE, DELETE ON "tess-storage-public".buckets TO administrator, authenticated, anonymous; @@ -72,7 +76,7 @@ DECLARE BEGIN INSERT INTO "tess-storage-public".files ( bucket_id, - key, + logical_key, content_hash, mime_type, size, diff --git a/graphql/server-test/__fixtures__/seed/shared-db-scope-storage/schema.sql b/graphql/server-test/__fixtures__/seed/shared-db-scope-storage/schema.sql index 6f35a6116c..47eab2b4c1 100644 --- a/graphql/server-test/__fixtures__/seed/shared-db-scope-storage/schema.sql +++ b/graphql/server-test/__fixtures__/seed/shared-db-scope-storage/schema.sql @@ -41,7 +41,8 @@ CREATE TABLE "shared-storage-public".files ( id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), database_id uuid NOT NULL, bucket_id uuid NOT NULL REFERENCES "shared-storage-public".buckets(id), - key text NOT NULL, + key text NOT NULL DEFAULT 'originals/' || gen_random_uuid()::text, + logical_key text, content_hash text NOT NULL, mime_type text NOT NULL, size bigint, @@ -55,6 +56,9 @@ CREATE TABLE "shared-storage-public".files ( UNIQUE (bucket_id, key) ); +CREATE INDEX ON "shared-storage-public".files (bucket_id, logical_key, content_hash) + WHERE status IN ('uploaded', 'processed'); + COMMENT ON TABLE "shared-storage-public".files IS E'@storageFiles\nStorage files table'; GRANT SELECT, INSERT, UPDATE, DELETE ON "shared-storage-public".buckets TO administrator, authenticated, anonymous; @@ -102,7 +106,7 @@ DECLARE v_row "shared-storage-public".files; BEGIN INSERT INTO "shared-storage-public".files ( - database_id, bucket_id, key, content_hash, mime_type, size, filename, is_public, previous_version_id + database_id, bucket_id, logical_key, content_hash, mime_type, size, filename, is_public, previous_version_id ) SELECT b.database_id, $1, $2, $3, $4, $5, $6, b.is_public, $9 FROM "shared-storage-public".buckets b diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql index 2de058ddd6..abb79d3e1b 100644 --- a/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql @@ -47,7 +47,8 @@ BEGIN 'CREATE TABLE IF NOT EXISTS %I.app_files ( id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), bucket_id uuid NOT NULL REFERENCES %I.app_buckets(id), - key text NOT NULL, + key text NOT NULL DEFAULT ''originals/'' || gen_random_uuid()::text, + logical_key text, content_hash text NOT NULL, mime_type text NOT NULL, size bigint, @@ -65,6 +66,10 @@ BEGIN 'COMMENT ON TABLE %I.app_files IS E''@storageFiles\nStorage files table''', schema_name); + EXECUTE format( + 'CREATE INDEX ON %I.app_files (bucket_id, logical_key, content_hash) WHERE status IN (''uploaded'', ''processed'')', + schema_name); + -- Grant CRUD to all roles EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON %I.app_buckets TO administrator, authenticated, anonymous', schema_name); EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON %I.app_files TO administrator, authenticated, anonymous', schema_name); @@ -90,7 +95,7 @@ BEGIN BEGIN INSERT INTO %I.app_files ( bucket_id, - key, + logical_key, content_hash, mime_type, size, diff --git a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts index 80996560e0..f818485fad 100644 --- a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts +++ b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts @@ -199,7 +199,7 @@ describe('database-scope upload surface', () => { // The tenant row names no endpoint: it inherits the platform row's, never an env default. expect(new URL(payload.uploadUrl).host).toBe('127.0.0.1:9000'); expect(payload.fileId).toBeTruthy(); - expect(payload.key).toBe(contentHash); + expect(payload.key).toMatch(/^originals\/[0-9a-f-]{36}$/); expect(payload.deduplicated).toBe(false); uploadUrl = payload.uploadUrl; diff --git a/graphql/server-test/__tests__/file-ref-urls.integration.test.ts b/graphql/server-test/__tests__/file-ref-urls.integration.test.ts index 235bf8a5fe..6b41f82484 100644 --- a/graphql/server-test/__tests__/file-ref-urls.integration.test.ts +++ b/graphql/server-test/__tests__/file-ref-urls.integration.test.ts @@ -6,7 +6,7 @@ * included, so it is never the authority for a URL. The files row is read by * the document's `id` as the reader; the source key comes from that row; and a * version is signed only when its key is a direct child of that row's own - * `.versions/`. + * `derived//`. * * pnpm test -- --testPathPattern=file-ref-urls */ @@ -136,8 +136,8 @@ describe('file-reference column URLs', () => { it('gives the narrowest version covering the width, else the source', async () => { const id = await addFile('pick/photo.png'); - const small = version('pick/photo.png.versions/small-0a1b2c3d.webp', 'small', 320); - const large = version('pick/photo.png.versions/large-4e5f6a7b.webp', 'large', 1280); + const small = version('derived/pick/photo.png/small-0a1b2c3d.webp', 'small', 320); + const large = version('derived/pick/photo.png/large-4e5f6a7b.webp', 'large', 1280); const row = await addProduct({ id, key: 'pick/photo.png', versions: [large, small] }); const at300 = await product(alice, row, 300); @@ -153,13 +153,13 @@ describe('file-reference column URLs', () => { it('never signs a key the document names outside its own file', async () => { const id = await addFile('forge/mine.png'); await addFile('forge/victim.png'); - const own = version('forge/mine.png.versions/small-0a1b2c3d.webp', 'small', 320); + const own = version('derived/forge/mine.png/small-0a1b2c3d.webp', 'small', 320); const forged = [ version('forge/victim.png', 'other-file', 400), - version('forge/victim.png.versions/small-0a1b2c3d.webp', 'other-files-version', 400), - version('forge/mine.png.versions/../victim.png', 'traversal', 400), - version('forge/mine.png.versions/nested/deep.webp', 'nested', 400), - version('forge/mine.png.versions/', 'prefix-only', 400), + version('derived/forge/victim.png/small-0a1b2c3d.webp', 'other-files-version', 400), + version('derived/forge/mine.png/../victim.png', 'traversal', 400), + version('derived/forge/mine.png/nested/deep.webp', 'nested', 400), + version('derived/forge/mine.png/', 'prefix-only', 400), version('forge/mine.png', 'source-as-version', 400) ]; // The document also lies about its own key: the row's key is what is signed. @@ -174,7 +174,7 @@ describe('file-reference column URLs', () => { it('resolves nothing for a document without a files row the reader can fetch', async () => { const requested = await addFile('dead/requested.png', 'requested'); - const noId = await addProduct({ key: 'pick/photo.png', versions: [version('pick/photo.png.versions/small-0a1b2c3d.webp', 'small', 320)] }); + const noId = await addProduct({ key: 'pick/photo.png', versions: [version('derived/pick/photo.png/small-0a1b2c3d.webp', 'small', 320)] }); const notLive = await addProduct({ id: requested, key: 'dead/requested.png' }); const unknown = await addProduct({ id: '00000000-0000-4000-8000-000000000000', key: 'pick/photo.png' }); const empty = await addProduct(null); @@ -192,7 +192,7 @@ describe('file-reference column URLs', () => { const hidden = await addProduct({ id: bobPrivateFileId, key: 'seeded-private-hash', - versions: [version('seeded-private-hash.versions/small-0a1b2c3d.webp', 'small', 320)] + versions: [version('derived/seeded-private-hash/small-0a1b2c3d.webp', 'small', 320)] }); const visible = await addProduct({ id: bobPublicFileId, key: 'seeded-public-hash' }); @@ -203,8 +203,8 @@ describe('file-reference column URLs', () => { it('resolves each element of an image[] column on its own', async () => { const one = await addFile('list/one.png'); await addFile('list/two.png'); - const mine = version('list/one.png.versions/small-0a1b2c3d.webp', 'small', 320); - const theirs = version('list/two.png.versions/small-0a1b2c3d.webp', 'small', 320); + const mine = version('derived/list/one.png/small-0a1b2c3d.webp', 'small', 320); + const theirs = version('derived/list/two.png/small-0a1b2c3d.webp', 'small', 320); const { rows } = await pg.query( `INSERT INTO ${schema}.albums (shots) VALUES (ARRAY(SELECT jsonb_array_elements($1::jsonb))::public.image[]) RETURNING id`, @@ -251,7 +251,7 @@ describe('file-reference column URLs', () => { it('falls back to the source when no version states a width', async () => { const id = await addFile('docs/deck.pdf'); - const preview = { name: 'preview', key: 'docs/deck.pdf.versions/preview-0a1b2c3d.png' }; + const preview = { name: 'preview', key: 'derived/docs/deck.pdf/preview-0a1b2c3d.png' }; const row = await addProduct(null, { id, key: 'docs/deck.pdf', versions: [preview] }); const got = await manual(row, 320); @@ -264,14 +264,14 @@ describe('file-reference column URLs', () => { it('never signs a key the document names outside its own file', async () => { const id = await addFile('docs/mine.pdf'); await addFile('docs/victim.pdf'); - const own = version('docs/mine.pdf.versions/page-0a1b2c3d.png', 'page', 320); + const own = version('derived/docs/mine.pdf/page-0a1b2c3d.png', 'page', 320); const row = await addProduct(null, { id, key: 'docs/victim.pdf', versions: [ version('docs/victim.pdf', 'other-file', 400), - version('docs/victim.pdf.versions/page-0a1b2c3d.png', 'other-files-version', 400), - version('docs/mine.pdf.versions/../victim.pdf', 'traversal', 400), + version('derived/docs/victim.pdf/page-0a1b2c3d.png', 'other-files-version', 400), + version('derived/docs/mine.pdf/../victim.pdf', 'traversal', 400), own ] }); @@ -297,7 +297,7 @@ describe('file-reference column URLs', () => { it('resolves each element of an upload[] column on its own', async () => { const one = await addFile('papers/one.pdf'); await addFile('papers/two.pdf'); - const theirs = version('papers/two.pdf.versions/page-0a1b2c3d.png', 'page', 320); + const theirs = version('derived/papers/two.pdf/page-0a1b2c3d.png', 'page', 320); const { rows } = await pg.query( `INSERT INTO ${schema}.albums (papers) VALUES (ARRAY(SELECT jsonb_array_elements($1::jsonb))::public.upload[]) RETURNING id`, @@ -341,7 +341,7 @@ describe('file-reference column URLs', () => { it('serves a public file from the public URL prefix, its versions too', async () => { const id = await addFile('open/logo.png', 'uploaded', alicePublicBucketId, true); - const small = version('open/logo.png.versions/small-0a1b2c3d.webp', 'small', 320); + const small = version('derived/open/logo.png/small-0a1b2c3d.webp', 'small', 320); const got = await urls(id, await addProduct({ id, key: 'open/logo.png', versions: [small] })); expect(got.source).toBe(`${cdn}/app-80a2eaaf-public-c7156cb19fe3/open/logo.png`); @@ -376,7 +376,7 @@ describe('file-reference column URLs', () => { VALUES ($1, 'team/plan.png', 'team/plan.png', 'image/png', 'plan.png', 'uploaded') RETURNING id`, [secondBucketId] ); - const small = version('team/plan.png.versions/small-0a1b2c3d.webp', 'small', 320); + const small = version('derived/team/plan.png/small-0a1b2c3d.webp', 'small', 320); const got = await product(alice, await addProduct({ id: rows[0].id, key: 'team/plan.png', versions: [small] }), 300); expect(pathOf(got.source)).toContain('team-80a2eaaf-private-0123456789ab/team/plan.png'); @@ -402,7 +402,7 @@ describe('file-reference column URLs', () => { const id = await addFile(key); await pg.query(`INSERT INTO ${schema}.products (title, photo) VALUES ($1, $2::jsonb)`, [ title, - JSON.stringify({ id, key, versions: [version(`${key}.versions/small-0a1b2c3d.webp`, 'small', 320)] }) + JSON.stringify({ id, key, versions: [version(`derived/${key}/small-0a1b2c3d.webp`, 'small', 320)] }) ]); } diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index 78f5ad2e5a..b79a424789 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -287,10 +287,10 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { * run: a row is written as `requested` and only becomes a dedup hit once the * confirm job has inspected the object and moved it to `uploaded`. */ - const confirmUploaded = async (contentHash: string): Promise => { + const confirmUploaded = async (fileId: string): Promise => { await pg.query( - `UPDATE "${aliceSchemas[0]}".app_files SET status = 'uploaded' WHERE content_hash = $1`, - [contentHash] + `UPDATE "${aliceSchemas[0]}".app_files SET status = 'uploaded' WHERE id = $1`, + [fileId] ); }; let teardown: () => Promise; @@ -419,7 +419,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.uploadUrl).toBeTruthy(); expect(new URL(payload.uploadUrl).host).toBe('127.0.0.1:9000'); expect(payload.fileId).toBeTruthy(); - expect(payload.key).toBe(contentHash); + expect(payload.key).toMatch(/^originals\/[0-9a-f-]{36}$/); expect(payload.deduplicated).toBe(false); expect(payload.expiresAt).toBeTruthy(); @@ -435,7 +435,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { it('should accept a PUT to the presigned URL', async () => { const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { headers: uploadHeaders }); expect(putRes.ok).toBe(true); - await confirmUploaded(contentHash); + await confirmUploaded(uploadHeaders['x-amz-meta-upload-id']); }); }); @@ -469,7 +469,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { const payload = data.uploadAppFile; expect(payload.uploadUrl).toBeTruthy(); expect(payload.fileId).toBeTruthy(); - expect(payload.key).toBe(contentHash); + expect(payload.key).toMatch(/^originals\/[0-9a-f-]{36}$/); expect(payload.deduplicated).toBe(false); expect(payload.expiresAt).toBeTruthy(); @@ -519,6 +519,34 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { // (stood in for here by confirmUploaded) moves it to `uploaded`, and only // an uploaded row is a dedup hit. A retry after a PUT that never ran must // get a URL again. + it('allocates separate objects for repeated caller-selected logical keys', async () => { + await pg.query(`INSERT INTO "${aliceSchemas[0]}".app_buckets + (key, type, is_public, allow_custom_keys, physical_name) + VALUES ('logical-slot', 'public', true, true, 'logical-slot-fixture')`); + try { + const key = 'derived/originals/known-object/thumb.webp'; + const request = async (text: string) => expectSuccess(await postGraphQL({ + query: UPLOAD_APP_FILE, + variables: { input: { + bucketKey: 'logical-slot', key, contentHash: await hashContent(text), + contentType: 'text/plain', size: Buffer.byteLength(text), filename: 'logical.txt' + } } + })).uploadAppFile; + const first = await request('first'); + const second = await request('second'); + expect(first.key).toMatch(/^originals\/[0-9a-f-]{36}$/); + expect(second.key).toMatch(/^originals\/[0-9a-f-]{36}$/); + expect(first.key).not.toBe(second.key); + expect(first.fileId).not.toBe(second.fileId); + expect(first.deduplicated).toBe(false); + expect(second.deduplicated).toBe(false); + } finally { + await pg.query(`DELETE FROM "${aliceSchemas[0]}".app_files WHERE bucket_id IN + (SELECT id FROM "${aliceSchemas[0]}".app_buckets WHERE key = 'logical-slot')`); + await pg.query(`DELETE FROM "${aliceSchemas[0]}".app_buckets WHERE key = 'logical-slot'`); + } + }); + it('issues a fresh upload URL when the earlier PUT never ran', async () => { const fileContent = `Never PUT: the client walked away (${crypto.randomUUID()}).`; const contentHash = await hashContent(fileContent); @@ -538,10 +566,11 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(retry.deduplicated).toBe(false); expect(retry.uploadUrl).toBeTruthy(); expect(retry.fileId).not.toBe(first.fileId); + expect(retry.key).not.toBe(first.key); const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain', { headers: retry.uploadHeaders }); expect(putRes.ok).toBe(true); - await confirmUploaded(contentHash); + await confirmUploaded(retry.fileId); const afterPut = expectSuccess(await postGraphQL({ query: UPLOAD_APP_FILE, variables: { input } })).uploadAppFile; expect(afterPut.deduplicated).toBe(true);