GitHub/outline
1
0
Fork 0
mirror of https://github.com/outline/outline.git synced 2026-08-03 13:27:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: support presigned PUT uploads for S3-compatible storage (#12748)

Browse Source 
* feat: support presigned PUT uploads for S3-compatible storage

Add `AWS_S3_UPLOAD_METHOD` config ("post" default / "put") so providers
that do not implement presigned POST (e.g. Cloudflare R2) can use
presigned PUT instead.

- BaseStorage.getPresignedPut() returns undefined by default; S3Storage
  overrides with PutObjectCommand + getSignedUrl.
- Content-Length is signed into the presigned PUT URL so S3 rejects
  uploads that do not match the declared size.
- API route and MCP tool branch on the config: only one upload method
  is returned per request.
- Frontend uses existing presignedPutUrl presence check — unchanged.
- When put is configured but unsupported (e.g. LocalStorage), a clear
  InvalidRequestError is returned.

Defaults to "post" — zero behavior change for existing deployments.

* refactor: return explicit upload mode instead of inferring from fields

Address review feedback from @tommoor: the client should not need to
infer the upload method from the presence of presigned-PUT-specific
fields. The API now returns an explicit `mode` ("put" | "post"), and the
PUT branch uses generic `url` / `headers` fields so the client has no
special knowledge of "presigned puts".

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* refactor: use this.getBucket() in presigned post/put methods

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Yinhao Huang
2026-06-24 21:05:16 -04:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 3999f2d902
commit 6286d08171
11 changed files with 470 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.sample
+5
View File
@@ -119,6 +119,11 @@ AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
AWS_S3_FORCE_PATH_STYLE=true
AWS_S3_ACL=private
# Which HTTP method to use for presigned uploads. "post" (default) uses the
# traditional S3 presigned POST with multipart form data. "put" uses a single
# PUT request with a presigned URL. Set this to "put" for providers like
# Cloudflare R2 that do not support presigned POST.
AWS_S3_UPLOAD_METHOD=post
# Optional CloudFront CDN for attachment downloads (uploads stay on S3).
# Requires a key pair for signed URLs; otherwise downloads fall back to S3.
app/utils/files.ts
+38 -26
View File
@@ -62,19 +62,7 @@ export const uploadFile = async (
invariant(response, "Response should be available");
const data = response.data;
const attachment = data.attachment;
const formData = new FormData();
for (const key in data.form) {
formData.append(key, data.form[key]);
}
// @ts-expect-error ts-migrate(2339) FIXME: Property 'blob' does not exist on type 'File | Blo... Remove this comment to see the full error message
if (file.blob) {
// @ts-expect-error ts-migrate(2339) FIXME: Property 'file' does not exist on type 'File | Blo... Remove this comment to see the full error message
formData.append("file", file.file);
} else {
formData.append("file", file);
}
const usePut = data.mode === "put";
// Using XMLHttpRequest instead of fetch because fetch doesn't support progress
const xhr = new XMLHttpRequest();
@@ -92,8 +80,6 @@ export const uploadFile = async (
size: file.size,
};
// Status 0 means the request never reached the server (network drop,
// CORS, abort) — log as a warning rather than an unhelpful "Error: 0".
if (xhr.status === 0) {
Logger.warn("File upload failed before response", extra);
return;
@@ -109,19 +95,45 @@ export const uploadFile = async (
resolve(xhr.readyState === 4 && xhr.status >= 200 && xhr.status < 400);
});
// Do not send credentials if uploading to a different origin, as the combination
// of CORS and cookies will cause preflight request failure. However S3-like storage
// on the same host can work with credentials.
if (data.uploadUrl.startsWith("/")) {
xhr.withCredentials = true;
if (usePut) {
xhr.open("PUT", data.url, true);
if (data.headers) {
for (const [key, value] of Object.entries(
data.headers as Record<string, string>
)) {
xhr.setRequestHeader(key, value);
}
}
// @ts-expect-error ts-migrate(2339) FIXME: Property 'blob' does not exist on type 'File | Blo...
xhr.send(file.blob ? file.file : file);
} else {
const parsed = new URL(data.uploadUrl);
const requiresPreflightRequest = parsed.origin !== window.location.origin;
xhr.withCredentials = !requiresPreflightRequest;
}
const formData = new FormData();
for (const key in data.form) {
formData.append(key, data.form[key]);
}
// @ts-expect-error ts-migrate(2339) FIXME: Property 'blob' does not exist on type 'File | Blo...
if (file.blob) {
// @ts-expect-error ts-migrate(2339) FIXME: Property 'file' does not exist on type 'File | Blo...
formData.append("file", file.file);
} else {
formData.append("file", file);
}
xhr.open("POST", data.uploadUrl, true);
xhr.send(formData);
// Do not send credentials if uploading to a different origin, as the
// combination of CORS and cookies will cause preflight request failure.
// However S3-like storage on the same host can work with credentials.
if (data.uploadUrl.startsWith("/")) {
xhr.withCredentials = true;
} else {
const parsed = new URL(data.uploadUrl);
const requiresPreflightRequest =
parsed.origin !== window.location.origin;
xhr.withCredentials = !requiresPreflightRequest;
}
xhr.open("POST", data.uploadUrl, true);
xhr.send(formData);
}
});
if (!success) {
server/env.ts
+10
View File
@@ -707,6 +707,16 @@ export class Environment {
@IsOptional()
public AWS_S3_ACL = environment.AWS_S3_ACL ?? "private";
/**
* Which HTTP method to use for presigned uploads to S3-compatible storage.
* "post" uses multipart form upload (traditional S3 presigned POST).
* "put" uses a single PUT request with a presigned URL (required for
* providers like Cloudflare R2 that do not support presigned POST).
*/
@IsIn(["put", "post"])
public AWS_S3_UPLOAD_METHOD =
this.toOptionalString(environment.AWS_S3_UPLOAD_METHOD) ?? "post";
/**
* Which file storage system to use
*/
server/routes/api/attachments/attachments.test.ts
+61
View File
@@ -1,5 +1,6 @@
import { randomUUID } from "node:crypto";
import { AttachmentPreset, CollectionPermission } from "@shared/types";
import env from "@server/env";
import { UserMembership } from "@server/models";
import Attachment from "@server/models/Attachment";
import {
@@ -165,6 +166,66 @@ describe("#attachments.create", () => {
expect(res.status).toEqual(200);
});
it("should return PUT data when AWS_S3_UPLOAD_METHOD is put", async () => {
const original = env.AWS_S3_UPLOAD_METHOD;
env.AWS_S3_UPLOAD_METHOD = "put";
try {
const user = await buildUser();
const res = await server.post("/api/attachments.create", user, {
body: {
name: "test.png",
contentType: "image/png",
size: 1000,
preset: AttachmentPreset.Avatar,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("put");
expect(body.data.url).toBeDefined();
expect(body.data.headers).toBeDefined();
expect(body.data.headers["Content-Type"]).toBeDefined();
expect(body.data.headers["Content-Length"]).toBe("1000");
expect(body.data.headers["Content-Disposition"]).toBeDefined();
expect(body.data.headers["Cache-Control"]).toBe("max-age=31557600");
expect(body.data.uploadUrl).toBeUndefined();
expect(body.data.form).toBeUndefined();
expect(body.data.attachment).toBeDefined();
} finally {
env.AWS_S3_UPLOAD_METHOD = original;
}
});
it("should return POST data when AWS_S3_UPLOAD_METHOD is post", async () => {
const original = env.AWS_S3_UPLOAD_METHOD;
env.AWS_S3_UPLOAD_METHOD = "post";
try {
const user = await buildUser();
const res = await server.post("/api/attachments.create", user, {
body: {
name: "test.png",
contentType: "image/png",
size: 1000,
preset: AttachmentPreset.Avatar,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("post");
expect(body.data.url).toBeUndefined();
expect(body.data.headers).toBeUndefined();
expect(body.data.uploadUrl).toBeDefined();
expect(body.data.form).toBeDefined();
expect(body.data.attachment).toBeDefined();
} finally {
env.AWS_S3_UPLOAD_METHOD = original;
}
});
it("should create expiring attachment using import preset", async () => {
const user = await buildUser();
const res = await server.post("/api/attachments.create", user, {
server/routes/api/attachments/attachments.ts
+50 -19
View File
@@ -3,6 +3,7 @@ import type { WhereOptions } from "sequelize";
import { randomUUID } from "node:crypto";
import { AttachmentPreset } from "@shared/types";
import { bytesToHumanReadable, getFileNameFromUrl } from "@shared/utils/files";
import env from "@server/env";
import { AttachmentValidation } from "@shared/validations";
import { createContext } from "@server/context";
import {
@@ -138,28 +139,58 @@ router.post(
userId: user.id,
});
const presignedPost = await FileStorage.getPresignedPost(
ctx,
key,
acl,
maxUploadSize,
contentType
);
const usePut = env.AWS_S3_UPLOAD_METHOD === "put";
ctx.body = {
data: {
uploadUrl: FileStorage.getUploadUrl(),
form: {
"Cache-Control": "max-age=31557600",
"Content-Type": contentType,
...presignedPost.fields,
if (usePut) {
const presignedPut = await FileStorage.getPresignedPut(
key,
acl,
size,
contentType
);
if (!presignedPut) {
throw InvalidRequestError(
`The current storage backend does not support PUT uploads. Set AWS_S3_UPLOAD_METHOD to "post" or use an S3-compatible storage provider.`
);
}
ctx.body = {
data: {
mode: "put",
url: presignedPut.url,
headers: presignedPut.headers,
attachment: {
...presentAttachment(attachment),
url: attachment.redirectUrl,
},
},
attachment: {
...presentAttachment(attachment),
url: attachment.redirectUrl,
};
} else {
const presignedPost = await FileStorage.getPresignedPost(
ctx,
key,
acl,
maxUploadSize,
contentType
);
ctx.body = {
data: {
mode: "post",
uploadUrl: FileStorage.getUploadUrl(),
form: {
"Cache-Control": "max-age=31557600",
"Content-Type": contentType,
...presignedPost.fields,
},
attachment: {
...presentAttachment(attachment),
url: attachment.redirectUrl,
},
},
},
};
};
}
}
);
server/storage/files/BaseStorage.test.ts
+13
View File
@@ -66,6 +66,19 @@ class MockStorage extends BaseStorage {
}
describe("BaseStorage", () => {
describe("getPresignedPut", () => {
it("should return undefined from default implementation", async () => {
const storage = new MockStorage();
const result = await storage.getPresignedPut(
"uploads/test/key",
"private",
1000000,
"image/png"
);
expect(result).toBeUndefined();
});
});
describe("storeFromUrl", () => {
let storage: MockStorage;
server/storage/files/BaseStorage.ts
+21
View File
@@ -40,6 +40,27 @@ export default abstract class BaseStorage {
contentType: string
): Promise<Partial<PresignedPost>>;
/**
* Returns a presigned PUT URL and the headers the client must send with the
* PUT request. Subclasses that support PUT-based uploads (e.g. S3) should
* override this method. Returns undefined by default, signalling the client
* should fall back to the POST flow.
*
* @param key The path to store the file at.
* @param acl The ACL to use.
* @param contentLength The exact content length in bytes, signed into the URL.
* @param contentType The content type of the file.
* @returns The presigned PUT URL and required headers, or undefined if not supported.
*/
public getPresignedPut(
_key: string,
_acl: string,
_contentLength: number,
_contentType: string
): Promise<{ url: string; headers: Record<string, string> } | undefined> {
return Promise.resolve(undefined);
}
/**
* Returns a promise that resolves with a stream for reading a file from the storage provider.
*
server/storage/files/S3Storage.ts
+53 -1
View File
@@ -7,6 +7,7 @@ import {
GetObjectCommand,
HeadObjectCommand,
CopyObjectCommand,
PutObjectCommand,
} from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import type { PresignedPostOptions } from "@aws-sdk/s3-presigned-post";
@@ -48,7 +49,7 @@ export default class S3Storage extends BaseStorage {
contentType = "image"
) {
const params: PresignedPostOptions = {
Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME as string,
Bucket: this.getBucket(),
Key: key,
Conditions: compact([
["content-length-range", 0, maxUploadSize],
@@ -66,6 +67,57 @@ export default class S3Storage extends BaseStorage {
return createPresignedPost(this.client, params);
}
/**
* Returns a presigned PUT URL with Content-Length signed into the request so
* S3 rejects uploads that do not match the declared size.
*
* @param key The path to store the file at.
* @param acl The ACL to use.
* @param contentLength The exact content length in bytes.
* @param contentType The content type of the file.
* @returns The presigned PUT URL and required headers.
*/
public async getPresignedPut(
key: string,
_acl: string,
contentLength: number,
contentType: string
): Promise<{ url: string; headers: Record<string, string> }> {
const contentDisposition = this.getContentDisposition(contentType);
const cacheControl = "max-age=31557600";
const command = new PutObjectCommand({
Bucket: this.getBucket(),
Key: key,
ContentType: contentType,
ContentLength: contentLength,
ContentDisposition: contentDisposition,
CacheControl: cacheControl,
...(env.AWS_S3_ACL && { ACL: env.AWS_S3_ACL as ObjectCannedACL }),
});
let url = await getSignedUrl(this.client, command, {
expiresIn: 3600,
});
if (env.AWS_S3_ACCELERATE_URL) {
url = url.replace(
env.AWS_S3_UPLOAD_BUCKET_URL,
env.AWS_S3_ACCELERATE_URL
);
}
return {
url,
headers: {
"Content-Type": contentType,
"Content-Length": String(contentLength),
"Content-Disposition": contentDisposition,
"Cache-Control": cacheControl,
},
};
}
private getPublicEndpoint(isServerUpload?: boolean) {
if (env.AWS_S3_ACCELERATE_URL) {
return env.AWS_S3_ACCELERATE_URL;
server/storage/files/__mocks__/index.ts
+12
View File
@@ -10,4 +10,16 @@ export default {
getSignedUrl: vi.fn().mockReturnValue("http://s3mock"),
getPresignedPost: vi.fn().mockReturnValue({}),
getPresignedPut: vi.fn().mockImplementation(
(_key: string, _acl: string, contentLength: number) => ({
url: "http://s3mock/presigned-put-url",
headers: {
"Content-Type": "application/octet-stream",
"Content-Length": String(contentLength),
"Content-Disposition": "attachment",
"Cache-Control": "max-age=31557600",
},
})
),
};
server/storage/files/presignedPut.test.ts
+143
View File
@@ -0,0 +1,143 @@
import { AttachmentPreset } from "@shared/types";
import env from "@server/env";
import { buildUser, buildDocument } from "@server/test/factories";
import { getTestServer } from "@server/test/support";
vi.mock("@server/storage/files");
const server = getTestServer();
describe("AWS_S3_UPLOAD_METHOD config", () => {
it("should return only PUT data when method is put", async () => {
const original = env.AWS_S3_UPLOAD_METHOD;
env.AWS_S3_UPLOAD_METHOD = "put";
try {
const user = await buildUser();
const document = await buildDocument({
teamId: user.teamId,
userId: user.id,
});
const res = await server.post("/api/attachments.create", user, {
body: {
name: "photo.jpg",
contentType: "image/jpeg",
size: 500000,
documentId: document.id,
preset: AttachmentPreset.DocumentAttachment,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("put");
expect(body.data.url).toBe("http://s3mock/presigned-put-url");
expect(body.data.headers).toEqual(
expect.objectContaining({
"Content-Type": expect.any(String),
"Content-Length": "500000",
"Content-Disposition": expect.any(String),
"Cache-Control": "max-age=31557600",
})
);
expect(body.data.uploadUrl).toBeUndefined();
expect(body.data.form).toBeUndefined();
expect(body.data.attachment.id).toBeDefined();
expect(body.data.attachment.url).toContain("/api/attachments.redirect");
} finally {
env.AWS_S3_UPLOAD_METHOD = original;
}
});
it("should return only POST data when method is post", async () => {
const original = env.AWS_S3_UPLOAD_METHOD;
env.AWS_S3_UPLOAD_METHOD = "post";
try {
const user = await buildUser();
const res = await server.post("/api/attachments.create", user, {
body: {
name: "photo.jpg",
contentType: "image/jpeg",
size: 500000,
preset: AttachmentPreset.Avatar,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("post");
expect(body.data.uploadUrl).toBeDefined();
expect(body.data.form).toBeDefined();
expect(body.data.form["Content-Type"]).toBe("image/jpeg");
expect(body.data.form["Cache-Control"]).toBe("max-age=31557600");
expect(body.data.url).toBeUndefined();
expect(body.data.headers).toBeUndefined();
expect(body.data.attachment).toBeDefined();
} finally {
env.AWS_S3_UPLOAD_METHOD = original;
}
});
it("should default to post method", async () => {
const user = await buildUser();
const res = await server.post("/api/attachments.create", user, {
body: {
name: "test.png",
contentType: "image/png",
size: 1000,
preset: AttachmentPreset.Avatar,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("post");
expect(body.data.uploadUrl).toBeDefined();
expect(body.data.form).toBeDefined();
expect(body.data.url).toBeUndefined();
});
it("should return correct headers for PUT with various content types", async () => {
const original = env.AWS_S3_UPLOAD_METHOD;
env.AWS_S3_UPLOAD_METHOD = "put";
try {
const user = await buildUser();
const document = await buildDocument({
teamId: user.teamId,
userId: user.id,
});
const res = await server.post("/api/attachments.create", user, {
body: {
name: "report.pdf",
contentType: "application/pdf",
size: 50000,
documentId: document.id,
preset: AttachmentPreset.DocumentAttachment,
},
});
expect(res.status).toEqual(200);
const body = await res.json();
expect(body.data.mode).toBe("put");
expect(body.data.url).toBeDefined();
expect(body.data.headers).toEqual(
expect.objectContaining({
"Content-Type": expect.any(String),
"Content-Length": "50000",
"Cache-Control": expect.any(String),
})
);
expect(body.data.attachment.contentType).toBe("application/pdf");
} finally {
env.AWS_S3_UPLOAD_METHOD = original;
}
});
});
server/tools/attachments.ts
+64 -30
View File
@@ -1,7 +1,8 @@
import { randomUUID } from "crypto";
import { z } from "zod";
import { type McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { ValidationError } from "@server/errors";
import env from "@server/env";
import { InvalidRequestError, ValidationError } from "@server/errors";
import { Attachment, Team } from "@server/models";
import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
import { authorize } from "@server/policies";
@@ -93,38 +94,71 @@ export function attachmentTools(server: McpServer, scopes: string[]) {
userId: user.id,
});
const presignedPost = await FileStorage.getPresignedPost(
ctx,
key,
acl,
maxUploadSize,
contentType
);
const usePut = env.AWS_S3_UPLOAD_METHOD === "put";
const uploadUrl = new URL(FileStorage.getUploadUrl(), team.url)
.href;
const form = {
"Cache-Control": "max-age=31557600",
"Content-Type": contentType,
...presignedPost.fields,
};
if (usePut) {
const presignedPut = await FileStorage.getPresignedPut(
key,
acl,
size,
contentType
);
// Build a ready-to-use curl command for the MCP client
const formArgs = Object.entries(form)
.map(([k, v]) => `-F '${k}=${v}'`)
.join(" ");
const curlCommand = `curl -X POST ${formArgs} -F 'file=@/path/to/file' '${uploadUrl}'`;
if (!presignedPut) {
throw InvalidRequestError(
`The current storage backend does not support PUT uploads. Set AWS_S3_UPLOAD_METHOD to "post" or use an S3-compatible storage provider.`
);
}
return success({
uploadUrl,
form,
maxUploadSize,
curlCommand,
attachment: pathToUrl(team, {
...presentAttachment(attachment),
url: attachment.redirectUrl,
}),
});
const curlCommand = `curl -X PUT ${Object.entries(presignedPut.headers)
.map(([k, v]) => `-H '${k}: ${v}'`)
.join(" ")} --data-binary '@/path/to/file' '${presignedPut.url}'`;
return success({
mode: "put",
url: presignedPut.url,
headers: presignedPut.headers,
maxUploadSize,
curlCommand,
attachment: pathToUrl(team, {
...presentAttachment(attachment),
url: attachment.redirectUrl,
}),
});
} else {
const presignedPost = await FileStorage.getPresignedPost(
ctx,
key,
acl,
maxUploadSize,
contentType
);
const uploadUrl = new URL(FileStorage.getUploadUrl(), team.url)
.href;
const form = {
"Cache-Control": "max-age=31557600",
"Content-Type": contentType,
...presignedPost.fields,
};
const formArgs = Object.entries(form)
.map(([k, v]) => `-F '${k}=${v}'`)
.join(" ");
const curlCommand = `curl -X POST ${formArgs} -F 'file=@/path/to/file' '${uploadUrl}'`;
return success({
mode: "post",
uploadUrl,
form,
maxUploadSize,
curlCommand,
attachment: pathToUrl(team, {
...presentAttachment(attachment),
url: attachment.redirectUrl,
}),
});
}
} catch (message) {
return error(message);
}