Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion graphile/graphile-presigned-url-plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Presigned URL upload plugin for PostGraphile v5.
- `requestUploadUrl` mutation — generates presigned PUT URLs for direct client-to-S3 upload
- `downloadUrl` computed field — presigned GET URLs for private files, public URLs for public files
- `maxBulkFiles` field on bucket types — the most files one bulk-upload call accepts (`storage_module.max_bulk_files`, default 100)
- Content-hash based S3 keys (SHA-256) with automatic deduplication — only against rows whose bytes are in S3, so a retry of a never-completed upload gets a fresh URL
- Independent server-allocated object keys (`originals/<UUID>`), with SHA-256 integrity and deduplication of confirmed content in the same logical slot
- Per-bucket MIME type and file size validation

## Usage
Expand All @@ -44,3 +44,23 @@ const preset = {
],
};
```

## Upload identity and retries

The optional upload input `key` names a **logical slot** (`files.logical_key`);
without it the logical slot is the SHA-256 content hash. It never chooses an S3
location. The generated recorder allocates a fresh `originals/<UUID>` for each
new request and returns it as `files.key`. The response `key`, projection `key`,
PUT signature and downloads all use that physical key.

A confirmed (`uploaded`/`processed`) row with the same bucket, logical key and
checksum is reused, without issuing a PUT URL. Pending or rejected attempts do
not deduplicate: a new request gets a new object and confirm job, and leaves the
older attempt to its existing lifecycle. Repeating a PUT with the same URL is a
retry of that same object; checksum and upload-id must still match its signature.

This requires the matching constructive-db storage generator with `logical_key`
and a server-allocated recorder result. Deploy the generator and pinned package
together; there is no fallback to signing the caller's logical key. Successful
logical replacement retains the existing database authorization policy; UUIDs
are object identities, not access control.
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

import { processSingleFile } from '../src/plugin';
import { generatePresignedPutUrl } from '../src/s3-signer';
import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types';

jest.mock('../src/s3-signer', () => ({
Expand All @@ -21,6 +22,8 @@ const BUCKET_ID = '33333333-3333-3333-3333-333333333333';
const OLD_FILE_ID = '44444444-4444-4444-4444-444444444444';
const NEW_FILE_ID = '55555555-5555-5555-5555-555555555555';
const KEY = 'index.html';
const OLD_KEY = `originals/${OLD_FILE_ID}`;
const NEW_KEY = `originals/${NEW_FILE_ID}`;

function storageConfig(overrides: Partial<StorageModuleConfig> = {}): StorageModuleConfig {
return {
Expand Down Expand Up @@ -60,15 +63,15 @@ function fakeTx(existingHash: string, deletable = true, existingStatus = 'upload
const txClient = {
async query(opts: { text: string; values: unknown[] }) {
queries.push(opts);
if (/^SELECT id, status/.test(opts.text)) {
if (/^SELECT id, (?:key|status)/.test(opts.text) && !/record_file\(/.test(opts.text)) {
// The probe is by (key, bucket, hash): a row carrying other bytes is not seen.
const probedHash = opts.values[2];
return { rows: probedHash === existingHash ? [{ id: OLD_FILE_ID, status: existingStatus }] : [] };
return { rows: probedHash === existingHash && ['uploaded', 'processed'].includes(existingStatus) ? [{ id: OLD_FILE_ID, key: OLD_KEY, status: existingStatus }] : [] };
}
if (/^DELETE FROM storage_public\.app_files/.test(opts.text)) {
return { rows: deletable ? [{ id: OLD_FILE_ID }] : [] };
}
if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID }] };
if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID, key: NEW_KEY }] };
throw new Error(`unexpected query: ${opts.text}`);
},
};
Expand Down Expand Up @@ -96,7 +99,7 @@ describe('custom-key upload of changed bytes', () => {

expect(result).toMatchObject({
fileId: NEW_FILE_ID,
key: KEY,
key: NEW_KEY,
deduplicated: false,
uploadUrl: 'https://s3.example.com/put',
});
Expand All @@ -116,23 +119,29 @@ describe('custom-key upload of changed bytes', () => {
const result = await upload(storageConfig(), txClient);

expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false });
expect(queries.map((q) => q.text.trim().split(/\s+/)[0])).toEqual(['SELECT', 'DELETE', 'SELECT']);
expect(queries[1].values).toEqual([OLD_FILE_ID]);
expect(queries.map((q) => q.text.trim().split(/\s+/)[0])).toEqual(['SELECT', 'SELECT']);
expect(result.key).toBe(NEW_KEY);
expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false);
});

it('refuses to restart a row the caller cannot delete', async () => {
it('retries independently even when the caller cannot delete an earlier attempt', async () => {
const { txClient, queries } = fakeTx('b'.repeat(64), false, 'requested');

const refused = upload(storageConfig(), txClient);
await expect(refused).rejects.toThrow(
`FILE_NOT_REPLACEABLE: file ${OLD_FILE_ID} at key ${KEY}`,
);
// A named error carries its code, so the server passes it through
// instead of masking it as INTERNAL_SERVER_ERROR.
await expect(refused).rejects.toMatchObject({ code: 'FILE_NOT_REPLACEABLE', errorClass: 'public' });
expect(queries.some((q) => /record_file\(/.test(q.text))).toBe(false);
await expect(upload(storageConfig(), txClient)).resolves.toMatchObject({ key: NEW_KEY, deduplicated: false });
expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false);
});

it.each(['a'.repeat(64), OLD_KEY, `derived/${OLD_KEY}/thumb.webp`])(
'never signs the attacker-selected logical key %s', async (logicalKey) => {
const { txClient } = fakeTx('a'.repeat(64));
const result = await processSingleFile(options, txClient, storageConfig(), 'db', bucket, s3, {
contentHash: 'b'.repeat(64), contentType: 'text/html', size: 16, key: logicalKey,
});
expect(result.key).toBe(NEW_KEY);
expect(result.file.key).toBe(NEW_KEY);
expect(generatePresignedPutUrl).toHaveBeenLastCalledWith(s3, NEW_KEY, 'text/html', 16, 900, 'b'.repeat(64), NEW_FILE_ID);
},
);

it('deduplicates identical bytes at the key without touching the row', async () => {
const { txClient, queries } = fakeTx('b'.repeat(64));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ describe('validateCustomKey', () => {
expect(validateCustomKey(key)).toMatch(/^INVALID_KEY:/);
});

// A file's rendered versions live under `<key>.versions/` and are served to
// whoever can read that file, so no file of its own may be stored there.
// The legacy `.versions/` suffix remains reserved in logical-key syntax;
// physical storage isolation no longer depends on this validation.
it.each(['photo.png.versions/secret.webp', 'a/b.versions/c/d.txt'])('rejects %s: the versions prefix is reserved', (key) => {
expect(validateCustomKey(key)).toMatch(/reserved/);
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ function fakeTx(rowKey: string, status: string) {
const txClient = {
async query(opts: { text: string; values: unknown[] }) {
queries.push(opts);
if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID }] };
if (/record_file\(/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID, key: `originals/${NEW_FILE_ID}` }] };
if (/^SELECT id/.test(opts.text)) {
return { rows: opts.values[0] === rowKey ? [{ id: OLD_FILE_ID, status }] : [] };
return { rows: opts.values[0] === rowKey && ['uploaded', 'processed'].includes(status) ? [{ id: OLD_FILE_ID, key: `originals/${OLD_FILE_ID}` }] : [] };
}
if (/^DELETE FROM storage_public\.app_files/.test(opts.text)) return { rows: [{ id: OLD_FILE_ID }] };
throw new Error(`unexpected query: ${opts.text}`);
Expand Down Expand Up @@ -125,7 +125,7 @@ describe.each([
});
expect(heads(s3)).toEqual([]);
const del = queries.find((q) => /^DELETE/.test(q.text));
expect(del?.values).toEqual([OLD_FILE_ID]);
expect(del).toBeUndefined();
},
);

Expand All @@ -152,8 +152,8 @@ describe('content-addressed upload whose hash a custom-key row carries', () => {

const result = await upload(txClient, s3);

expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false, key: HASH });
expect(queries[0].values).toEqual([HASH, BUCKET_ID]);
expect(result).toMatchObject({ fileId: NEW_FILE_ID, deduplicated: false, key: `originals/${NEW_FILE_ID}` });
expect(queries[0].values).toEqual([HASH, BUCKET_ID, HASH]);
expect(queries.some((q) => /^DELETE/.test(q.text))).toBe(false);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,14 @@ function input() {
describe('recordManagedFile', () => {
it('calls the generated recorder with only base object facts', async () => {
const query = jest.fn().mockResolvedValue({
rows: [{ id: '33333333-3333-3333-3333-333333333333' }],
rows: [{ id: '33333333-3333-3333-3333-333333333333', key: 'originals/server-generated' }],
});

await expect(recordManagedFile({ query }, storageConfig(), input())).resolves.toBe(
'33333333-3333-3333-3333-333333333333',
);
await expect(recordManagedFile({ query }, storageConfig(), input())).resolves.toEqual({ id: '33333333-3333-3333-3333-333333333333', key: 'originals/server-generated' });

expect(query).toHaveBeenCalledWith({
text: expect.stringContaining(
'SELECT id FROM storage_private.app_files_record_file(bucket_id := $1::uuid, key := $2::text',
'SELECT id, key FROM storage_private.app_files_record_file(bucket_id := $1::uuid, key := $2::text',
),
values: [
input().bucketId,
Expand All @@ -76,7 +74,7 @@ describe('recordManagedFile', () => {
});

it('appends a supported path argument by name and never a version link', async () => {
const query = jest.fn().mockResolvedValue({ rows: [{ id: 'file-id' }] });
const query = jest.fn().mockResolvedValue({ rows: [{ id: 'file-id', key: 'originals/server-generated' }] });

await recordManagedFile(
{ query },
Expand All @@ -103,6 +101,20 @@ describe('recordManagedFile', () => {
});
});

it.each([
undefined,
{ id: 'file-id' },
{ id: 'file-id', key: '' },
{ key: 'originals/server-generated' },
{ id: 'file-id', key: input().key },
])('refuses an invalid or echoed recorder identity: %p', async (row) => {
const query = jest.fn().mockResolvedValue({ rows: row ? [row] : [] });
await expect(recordManagedFile({ query }, storageConfig(), input())).rejects.toMatchObject({
code: 'INTERNAL_ERROR',
errorClass: 'internal',
});
});

it('fails loudly when a path is requested on a module without path shares', async () => {
const query = jest.fn();

Expand Down
3 changes: 1 addition & 2 deletions graphile/graphile-presigned-url-plugin/src/custom-key.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ export function validateCustomKey(key: string): string | null {
if (key.includes('\0')) {
return 'INVALID_KEY: null bytes not allowed';
}
// `<key>.versions/` is where a file's rendered versions are stored, and they
// are served to whoever can read that file: no file of its own may live there.
// Preserve the existing logical-key syntax restriction on the legacy suffix.
if (key.includes('.versions/')) {
return 'INVALID_KEY: ".versions/" is reserved for a file\'s rendered versions';
}
Expand Down
8 changes: 4 additions & 4 deletions graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* (`FILE_REF_DOMAINS`: `image`, `upload`), or an array of one. It holds a
* document that projects a files row — `id`, `key`, `bucket_id` — and, when a
* lane has rendered versions beside the source, lists them under `versions`
* (`<key>.versions/<name>-<hash>.<ext>`). The column reaches GraphQL as JSON,
* (`derived/<key>/<name>-<hash>.<ext>`). The column reaches GraphQL as JSON,
* so without these fields a client has no URL to fetch the file with.
*
* For each such column `photo` on a table type this adds
Expand All @@ -23,7 +23,7 @@
* 2. the source key, bucket, filename and publicness come from that row,
* never from the document;
* 3. a version is signed only when its key is a direct child of that row's
* own `<key>.versions/` prefix. A key the document merely names is never
* own `derived/<key>/` prefix. A key the document merely names is never
* signed.
*
* Files rows are read once per request, not once per document: every document
Expand Down Expand Up @@ -88,11 +88,11 @@ type PgClientLike = {

/**
* Whether `variantKey` is an object a lane wrote beside the file at
* `sourceKey`: a direct child of `<sourceKey>.versions/`.
* `sourceKey`: a direct child of `derived/<sourceKey>/`.
*/
export function isVersionKeyOf(sourceKey: string, variantKey: unknown): variantKey is string {
if (typeof variantKey !== 'string' || !sourceKey) return false;
const prefix = `${sourceKey}.versions/`;
const prefix = `derived/${sourceKey}/`;
if (!variantKey.startsWith(prefix)) return false;
const name = variantKey.slice(prefix.length);
return name.length > 0 && !name.includes('/') && name !== '.' && name !== '..';
Expand Down
Loading
Loading