From f1d3f555d72c6b1547138f57f4e7edfd17f88d1b Mon Sep 17 00:00:00 2001 From: KevinAndrewDong <2970752254@qq.com> Date: Thu, 1 Oct 2026 10:25:02 +0800 Subject: [PATCH 1/3] fix(utils): prevent SSRF in untrusted URL downloads --- .../modules/misc/misc.service.ssrf.spec.ts | 84 ++++ apps/api/src/modules/misc/misc.service.ts | 4 +- packages/utils/package.json | 9 +- packages/utils/src/index.ts | 1 - packages/utils/src/safe-fetch.test.ts | 397 ++++++++++++++++++ packages/utils/src/safe-fetch.ts | 87 ++++ packages/utils/src/scrape-weblink.ts | 3 +- packages/utils/src/server.ts | 3 + packages/utils/tsconfig.json | 6 +- pnpm-lock.yaml | 15 + 10 files changed, 602 insertions(+), 7 deletions(-) create mode 100644 apps/api/src/modules/misc/misc.service.ssrf.spec.ts create mode 100644 packages/utils/src/safe-fetch.test.ts create mode 100644 packages/utils/src/safe-fetch.ts create mode 100644 packages/utils/src/server.ts diff --git a/apps/api/src/modules/misc/misc.service.ssrf.spec.ts b/apps/api/src/modules/misc/misc.service.ssrf.spec.ts new file mode 100644 index 0000000000..493a7df821 --- /dev/null +++ b/apps/api/src/modules/misc/misc.service.ssrf.spec.ts @@ -0,0 +1,84 @@ +import { ParamsError } from '@refly/errors'; +import { safeFetch, scrapeWeblink } from '@refly/utils/server'; +import { MiscService } from './misc.service'; + +jest.mock('@refly/utils/server', () => ({ safeFetch: jest.fn(), scrapeWeblink: jest.fn() })); +jest.mock('@refly/utils', () => ({})); +jest.mock('@paralleldrive/cuid2', () => ({ createId: jest.fn() })); +jest.mock('sharp', () => jest.fn()); +jest.mock('../common/prisma.service', () => ({ PrismaService: class {} })); +jest.mock('../common/redis.service', () => ({ RedisService: class {} })); +jest.mock('../common/object-storage', () => ({ + ObjectStorageService: class {}, + OSS_EXTERNAL: 'external', + OSS_INTERNAL: 'internal', +})); +jest.mock('../knowledge/parsers/pandoc.parser', () => ({ PandocParser: class {} })); +jest.mock('../../utils', () => ({ + QUEUE_IMAGE_PROCESSING: 'images', + QUEUE_CLEAN_STATIC_FILES: 'cleanup', +})); + +describe('MiscService untrusted downloads', () => { + const guardedFetch = jest.mocked(safeFetch); + const guardedScrape = jest.mocked(scrapeWeblink); + const directFetch = jest.spyOn(globalThis, 'fetch'); + let service: MiscService; + + beforeEach(() => { + jest.resetAllMocks(); + directFetch.mockRejectedValue(new Error('Direct network access is forbidden in this test')); + service = Object.create(MiscService.prototype) as MiscService; + }); + afterAll(() => directFetch.mockRestore()); + + it('routes metadata scraping through the guarded scraper', async () => { + const result = { title: 'Public title', description: 'Public description', image: 'image.png' }; + guardedScrape.mockResolvedValue(result); + expect(await service.scrapeWeblink({ url: 'https://public.example' })).toEqual(result); + expect(guardedScrape).toHaveBeenCalledWith('https://public.example'); + }); + + it('propagates the guard rejection in both paths without uploading', async () => { + const blocked = new ParamsError('URL must resolve to a public HTTP(S) destination'); + guardedScrape.mockRejectedValue(blocked); + guardedFetch.mockRejectedValue(blocked); + const upload = jest.spyOn(service, 'uploadFile'); + await expect(service.scrapeWeblink({ url: 'http://127.0.0.1' })).rejects.toBe(blocked); + await expect( + service.dumpFileFromURL({ uid: 'user' }, { url: 'http://127.0.0.1' }), + ).rejects.toBe(blocked); + expect(upload).not.toHaveBeenCalled(); + }); + + it('keeps binary bytes, filename, MIME type and upload context', async () => { + const bytes = new Uint8Array([0, 255, 1, 128]); + // Only the Response methods used by this consumer are needed here. + guardedFetch.mockResolvedValue({ + arrayBuffer: async () => bytes.buffer, + headers: new Headers({ 'content-type': 'image/png' }), + } as Awaited>); + const uploaded = { url: 'https://files.example/image.png', storageKey: 'image' }; + const upload = jest.spyOn(service, 'uploadFile').mockResolvedValue(uploaded); + expect( + await service.dumpFileFromURL( + { uid: 'user' }, + { + url: 'https://public.example/image?token=removed', + entityId: 'resource', + entityType: 'resource', + }, + ), + ).toEqual(uploaded); + expect(guardedFetch).toHaveBeenCalledWith('https://public.example/image?token=removed'); + expect(upload).toHaveBeenCalledWith( + { uid: 'user' }, + { + file: { buffer: Buffer.from(bytes), mimetype: 'image/png', originalname: 'image.png' }, + entityId: 'resource', + entityType: 'resource', + visibility: 'private', + }, + ); + }); +}); diff --git a/apps/api/src/modules/misc/misc.service.ts b/apps/api/src/modules/misc/misc.service.ts index b97c1ca565..d7976c5b96 100644 --- a/apps/api/src/modules/misc/misc.service.ts +++ b/apps/api/src/modules/misc/misc.service.ts @@ -28,11 +28,11 @@ import path from 'node:path'; import { ConfigService } from '@nestjs/config'; import { omit, - scrapeWeblink, getSafeMimeType, runModuleInitWithTimeoutAndRetry, safeParseJSON, } from '@refly/utils'; +import { safeFetch, scrapeWeblink } from '@refly/utils/server'; import { QUEUE_IMAGE_PROCESSING, QUEUE_CLEAN_STATIC_FILES, streamToBuffer } from '../../utils'; import { CanvasNotFoundError, @@ -174,7 +174,7 @@ export class MiscService implements OnModuleInit { }, ): Promise { const { url, entityId, entityType, visibility = 'private' } = param; - const res = await fetch(url); + const res = await safeFetch(url); const buffer = await res.arrayBuffer(); // Extract filename from URL, removing query parameters and fragments diff --git a/packages/utils/package.json b/packages/utils/package.json index 09b3fed21f..9d54ee07b3 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -6,11 +6,13 @@ "types": "src/index.ts", "exports": { ".": "./src/index.ts", - "./token": "./src/token.ts" + "./token": "./src/token.ts", + "./server": "./src/server.ts" }, "typesVersions": { "*": { - "token": ["src/token.ts"] + "token": ["src/token.ts"], + "server": ["src/server.ts"] } }, "scripts": { @@ -25,11 +27,13 @@ "dependencies": { "@paralleldrive/cuid2": "^2.2.2", "@refly/common-types": "workspace:*", + "@refly/errors": "workspace:*", "@refly/openapi-schema": "workspace:*", "@tiptap/pm": "2.7.4", "cheerio": "~1.0.0", "clsx": "^2.1.0", "deepmerge": "^4.3.1", + "ipaddr.js": "2.2.0", "linkify-it": "^5.0.0", "markdown-it": "~14.1.0", "md5": "^2.3.0", @@ -39,6 +43,7 @@ "tailwind-merge": "^2.2.2", "kitoken": "^0.10.1", "tlds": "^1.255.0", + "undici": "6.29.0", "uuid": "^9.0.0", "y-prosemirror": "~1.2.6", "y-protocols": "~1.0.6", diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 81f9db567f..a3e2102faa 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -6,7 +6,6 @@ export * from './parse'; export * from './editor'; export * from './typesafe'; export * from './url'; -export * from './scrape-weblink'; export * from './models'; export * from './event-emitter/editor'; export * from './event-emitter/model'; diff --git a/packages/utils/src/safe-fetch.test.ts b/packages/utils/src/safe-fetch.test.ts new file mode 100644 index 0000000000..2e9dd5972e --- /dev/null +++ b/packages/utils/src/safe-fetch.test.ts @@ -0,0 +1,397 @@ +import { createServer } from 'node:http'; +import net, { type LookupFunction, type TcpNetConnectOpts } from 'node:net'; +import type { LookupAddress, LookupAllOptions } from 'node:dns'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ParamsError } from '@refly/errors'; +import ipaddr from 'ipaddr.js'; +import { Agent, Response, fetch } from 'undici'; +import { safeFetch } from './safe-fetch'; +import { scrapeWeblink } from './scrape-weblink'; + +type Lookup = ( + hostname: string, + options: LookupAllOptions, + callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void, +) => void; +const network = vi.hoisted(() => ({ lookup: vi.fn(), fetch: vi.fn() })); +const connection = vi.hoisted(() => ({ + lookup: undefined as unknown as LookupFunction, + agent: undefined as unknown as Agent, +})); +vi.mock('node:dns', () => ({ lookup: network.lookup })); +vi.mock('undici', async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + fetch: network.fetch, + Agent: class extends original.Agent { + constructor(options: Agent.Options) { + super(options); + if (typeof options.connect === 'object' && 'lookup' in options.connect) { + connection.lookup = options.connect.lookup!; + } + connection.agent = this; + } + }, + }; +}); + +async function resolveAtSocket(hostname: string): Promise { + return new Promise((resolve, reject) => { + connection.lookup(hostname, { all: false }, (error, address) => { + if (error) reject(error); + else resolve(address as string); + }); + }); +} + +async function expectBlocked(input: string) { + try { + await safeFetch(input); + expect.fail('Expected the destination to be rejected'); + } catch (error) { + expect(error).toBeInstanceOf(ParamsError); + const blocked = error as ParamsError & { cause?: unknown }; + expect(blocked.code).toBe('E0003'); + expect(blocked.message).toBe('URL must resolve to a public HTTP(S) destination'); + expect(blocked.cause).toBeUndefined(); + expect(`${blocked.message}\n${blocked.stack}`).not.toContain('PRIVATE_SENTINEL'); + } +} + +beforeEach(() => { + network.lookup.mockReset().mockImplementation((_hostname, options, callback) => { + expect(options).toEqual({ all: true, verbatim: true }); + callback(null, [{ address: '93.184.216.34', family: 4 }]); + }); + network.fetch.mockReset().mockImplementation(async (input, options) => { + expect(options?.redirect).toBe('manual'); + expect(options?.dispatcher).toBe(connection.agent); + const url = new URL(String(input)); + const host = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; + if (!net.isIP(host)) await resolveAtSocket(host); + return new Response('public content'); + }); +}); +afterAll(() => connection.agent.close()); + +describe('safeFetch destination policy', () => { + it.each([ + 'http://93.184.216.34', + 'https://93.184.216.34', + 'http://[2606:4700:4700::1111]', + 'https://[2606:4700:4700::1111]', + 'https://[::ffff:93.184.216.34]', + 'http://public.example', + 'https://public.example', + ])('allows public HTTP(S): %s', async (url) => { + expect(await (await safeFetch(url)).text()).toBe('public content'); + }); + + it.each([ + '127.0.0.1', + '127.9.8.7', + '127.1', + '2130706433', + '0x7f000001', + '0177.0.0.1', + '10.0.0.1', + '10.255.255.255', + '172.16.0.1', + '172.31.255.255', + '192.168.0.1', + '169.254.169.254', + '169.254.1.1', + '0.0.0.0', + '0.1.2.3', + '100.64.0.1', + '224.0.0.1', + '239.255.255.255', + '240.0.0.1', + '255.255.255.255', + '198.18.0.1', + '[::1]', + '[::]', + '[fc00::1]', + '[fdff::1]', + '[fe80::1]', + '[febf::1]', + '[fec0::1]', + '[ff02::1]', + '[::ffff:127.0.0.1]', + '[::ffff:10.0.0.1]', + '[::ffff:169.254.169.254]', + '[::ffff:0.0.0.0]', + '[::10.0.0.1]', + '[64:ff9b::a00:1]', + '[2002:0a00:0001::1]', + ])('rejects non-public addresses and alternative forms: %s', async (host) => { + await expectBlocked(`http://${host}/PRIVATE_SENTINEL`); + expect(network.fetch).not.toHaveBeenCalled(); + }); + + it.each(['file:', 'ftp:', 'gopher:', 'data:', 'javascript:'])('rejects %s', async (scheme) => { + await expectBlocked(`${scheme}//public.example/PRIVATE_SENTINEL`); + expect(network.fetch).not.toHaveBeenCalled(); + }); + + it.each([ + '', + 'PRIVATE_SENTINEL', + 'http://', + 'http://[::1', + 'http://%ff/PRIVATE_SENTINEL', + 'http://[fe80::1%25lo0]', + 'https://PRIVATE_SENTINEL:password@public.example', + `https://public.example/${'PRIVATE_SENTINEL'.repeat(1000)}`, + null, + undefined, + 123, + {}, + ['https://public.example'], + ])('rejects invalid syntax, encodings, types and oversized URLs: %s', async (input) => { + await expectBlocked(input as string); + expect(network.fetch).not.toHaveBeenCalled(); + }); + + it('does not invoke input conversion hooks', async () => { + const convert = vi.fn(() => { + throw new Error('PRIVATE_SENTINEL'); + }); + await expectBlocked({ toString: convert } as unknown as string); + expect(convert).not.toHaveBeenCalled(); + }); + + it.each( + [ + [{ address: '127.0.0.1', family: 4 }], + [{ address: '::1', family: 6 }], + [ + { address: '93.184.216.34', family: 4 }, + { address: '10.0.0.1', family: 4 }, + ], + [ + { address: '2606:4700:4700::1111', family: 6 }, + { address: '::ffff:127.0.0.1', family: 6 }, + ], + [{ address: 'PRIVATE_SENTINEL', family: 4 }], + [{ address: '2606:4700:4700::1111%eth-0', family: 6 }], + [{ address: '2606:4700:4700::1111%eth0', family: 6 }], + [], + ].map((answers) => ({ answers })), + )('rejects private, mixed, invalid or empty DNS answers: %j', async ({ answers }) => { + network.lookup.mockImplementation((_hostname, _options, callback) => callback(null, answers)); + await expectBlocked('https://PRIVATE_SENTINEL.example'); + }); + + it('checks all public answers and selects one for the socket', async () => { + network.lookup.mockImplementation((_host, _options, callback) => + callback(null, [ + { address: '93.184.216.34', family: 4 }, + { address: '2606:4700:4700::1111', family: 6 }, + ]), + ); + expect(await resolveAtSocket('public.example')).toBe('93.184.216.34'); + expect(network.lookup).toHaveBeenCalledTimes(1); + }); + + it('returns only checked candidates for Node dual-stack connection fallback', async () => { + const answers = [ + { address: '93.184.216.34', family: 4 }, + { address: '2606:4700:4700::1111', family: 6 }, + ]; + network.lookup.mockImplementation((_host, _options, callback) => callback(null, answers)); + const candidates = await new Promise((resolve, reject) => { + connection.lookup('public.example', { all: true }, (error, addresses) => { + if (error) reject(error); + else resolve(addresses as LookupAddress[]); + }); + }); + expect(candidates).toEqual(answers); + expect(network.lookup).toHaveBeenCalledTimes(1); + }); + + it('normalizes DNS and connection failures without unsafe causes', async () => { + network.lookup.mockImplementation((_host, _options, callback) => { + callback(new Error('PRIVATE_SENTINEL DNS failure'), []); + }); + await expectBlocked('https://public.example'); + network.fetch.mockRejectedValue( + Object.assign(new Error('PRIVATE_SENTINEL'), { + cause: new Error('PRIVATE_SENTINEL resolved address'), + }), + ); + await expectBlocked('https://public.example'); + }); + + it('normalizes IP parser failures inside the socket lookup callback', async () => { + const parse = vi.spyOn(ipaddr, 'process').mockImplementation(() => { + throw new Error('PRIVATE_SENTINEL'); + }); + try { + await expect(resolveAtSocket('public.example')).rejects.toBeInstanceOf(ParamsError); + await expectBlocked('https://public.example'); + } finally { + parse.mockRestore(); + } + }); +}); + +describe('redirects', () => { + it.each([301, 302, 303, 307, 308])( + 'revalidates status %i and relative Location', + async (status) => { + const redirect = new Response('redirect body', { status, headers: { location: '/next' } }); + const cancel = vi.spyOn(redirect.body!, 'cancel'); + network.fetch.mockResolvedValueOnce(redirect); + expect(await (await safeFetch('https://public.example/start')).text()).toBe('public content'); + expect(String(network.fetch.mock.calls[1][0])).toBe('https://public.example/next'); + expect(cancel).toHaveBeenCalledOnce(); + }, + ); + + it.each([ + 'http://127.0.0.1/PRIVATE_SENTINEL', + '//169.254.169.254/PRIVATE_SENTINEL', + 'http://[::ffff:127.0.0.1]/PRIVATE_SENTINEL', + 'file:///PRIVATE_SENTINEL', + 'http://[PRIVATE_SENTINEL', + `https://public.example/${'PRIVATE_SENTINEL'.repeat(1000)}`, + ])('blocks unsafe redirects before the next request: %s', async (location) => { + network.fetch.mockResolvedValueOnce(new Response(null, { status: 302, headers: { location } })); + await expectBlocked('https://public.example'); + expect(network.fetch).toHaveBeenCalledTimes(1); + }); + + it('checks DNS for a redirected hostname', async () => { + network.fetch.mockResolvedValueOnce( + new Response(null, { + status: 302, + headers: { location: 'https://internal.example' }, + }), + ); + network.lookup.mockImplementation((_host, _options, callback) => { + callback(null, [{ address: '127.0.0.1', family: 4 }]); + }); + await expectBlocked('https://public.example'); + expect(network.lookup.mock.calls[0][0]).toBe('internal.example'); + }); + + it('rejects a later private hop', async () => { + network.fetch + .mockResolvedValueOnce(new Response(null, { status: 302, headers: { location: '/two' } })) + .mockResolvedValueOnce( + new Response(null, { + status: 307, + headers: { location: 'http://169.254.169.254/PRIVATE_SENTINEL' }, + }), + ); + await expectBlocked('https://public.example/one'); + expect(network.fetch).toHaveBeenCalledTimes(2); + }); + + it('caps a redirect loop at five redirects', async () => { + network.fetch.mockImplementation( + async () => + new Response(null, { + status: 302, + headers: { location: '/loop' }, + }), + ); + await expectBlocked('https://public.example/loop'); + expect(network.fetch).toHaveBeenCalledTimes(6); + }); + + it('preserves a non-redirect response and a redirect without Location', async () => { + network.fetch + .mockResolvedValueOnce(new Response('missing', { status: 404 })) + .mockResolvedValueOnce(new Response('no location', { status: 302 })); + expect((await safeFetch('https://public.example')).status).toBe(404); + expect(await (await safeFetch('https://public.example')).text()).toBe('no location'); + }); +}); + +describe('consumers and connection binding', () => { + it('preserves HTML metadata and binary bytes', async () => { + network.fetch + .mockResolvedValueOnce( + new Response( + 'Public title' + + '', + ), + ) + .mockResolvedValueOnce(new Response(new Uint8Array([0, 255, 1, 128]))); + expect(await scrapeWeblink('https://public.example')).toEqual({ + title: 'Public title', + description: 'Public description', + image: 'https://public.example/image.png', + }); + expect( + new Uint8Array(await (await safeFetch('https://public.example/image.png')).arrayBuffer()), + ).toEqual(new Uint8Array([0, 255, 1, 128])); + }); + + it('blocks the original localhost reproduction without contacting the service', async () => { + let hits = 0; + const server = createServer((_request, response) => { + hits++; + response.end('INTERNAL_SECRET_TITLE'); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const port = (server.address() as net.AddressInfo).port; + await expect(scrapeWeblink(`http://127.0.0.1:${port}`)).rejects.toBeInstanceOf(ParamsError); + expect(hits).toBe(0); + expect(network.fetch).not.toHaveBeenCalled(); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it('uses the validated socket lookup without a second DNS resolution', async () => { + const server = createServer((_request, response) => response.end('bound connection')); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as net.AddressInfo).port; + const approved: string[] = []; + network.lookup + .mockImplementationOnce((_host, _options, callback) => { + callback(null, [{ address: '93.184.216.34', family: 4 }]); + }) + .mockImplementation((_host, _options, callback) => { + callback(null, [{ address: '127.0.0.1', family: 4 }]); + }); + // Intercept only TCP dialing: require the real undici Agent's lookup and + // reroute its approved IP to our local fixture. No public network is used. + const dial = vi.spyOn(net, 'connect').mockImplementation(((options: TcpNetConnectOpts) => { + const socket = new net.Socket(); + expect(options.lookup).toBe(connection.lookup); + expect(options.autoSelectFamily).toBe(true); + options.lookup!(options.host!, { all: true }, (error, addresses) => { + if (error) { + socket.destroy(error); + return; + } + approved.push(...(addresses as LookupAddress[]).map(({ address }) => address)); + socket.connect({ host: '127.0.0.1', port }); + }); + return socket; + }) as typeof net.connect); + const original = await vi.importActual('undici'); + network.fetch.mockImplementation(original.fetch); + try { + expect(await (await safeFetch(`http://binding.example:${port}`)).text()).toBe( + 'bound connection', + ); + expect(approved).toEqual(['93.184.216.34']); + expect(network.lookup).toHaveBeenCalledTimes(1); + // A new real connection seeing the rebound answer is rejected. + await expectBlocked(`http://binding.example:${port}`); + expect(network.lookup).toHaveBeenCalledTimes(2); + expect(approved).toEqual(['93.184.216.34']); + } finally { + dial.mockRestore(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +}); diff --git a/packages/utils/src/safe-fetch.ts b/packages/utils/src/safe-fetch.ts new file mode 100644 index 0000000000..34a19485b0 --- /dev/null +++ b/packages/utils/src/safe-fetch.ts @@ -0,0 +1,87 @@ +import { lookup } from 'node:dns'; +import { isIP } from 'node:net'; +import { ParamsError } from '@refly/errors'; +import ipaddr from 'ipaddr.js'; +import { Agent, fetch } from 'undici'; + +const publicIPv6 = ipaddr.IPv6.parse('2000::'); +const redirects = new Set([301, 302, 303, 307, 308]); +const maxRedirects = 5; +const errorMessage = 'URL must resolve to a public HTTP(S) destination'; + +function isPublicAddress(address: string): boolean { + try { + if (!isIP(address) || address.includes('%')) { + return false; + } + // Normalize IPv4-mapped IPv6 before applying the IPv4 range policy. + const ip = ipaddr.process(address); + return ip.range() === 'unicast' && (ip.kind() === 'ipv4' || ip.match(publicIPv6, 3)); + } catch { + // A parser failure in the asynchronous DNS callback must also fail closed. + return false; + } +} + +const dispatcher = new Agent({ + // Untrusted servers must not share a reusable HTTP/1 connection across requests. + pipelining: 0, + connect: { + autoSelectFamily: true, + lookup(hostname, options, callback) { + // This is the socket's lookup: only checked addresses reach connect(). + // Check all answers so a mixed public/private DNS response fails closed. + lookup(hostname, { all: true, verbatim: true }, (error, addresses) => { + if ( + error || + !addresses?.length || + addresses.some(({ address }) => !isPublicAddress(address)) + ) { + callback(new ParamsError(errorMessage), '', 0); + return; + } + if (options.all) { + callback(null, addresses); + } else { + callback(null, addresses[0].address, addresses[0].family); + } + }); + }, + }, +}); + +/** Server-only GET for untrusted URLs, including every redirect destination. */ +export async function safeFetch(input: string) { + try { + // Reject coercion of non-string input and bound URL parsing work. + if (typeof input !== 'string' || input.length > 8192) { + throw new ParamsError(errorMessage); + } + let url = new URL(input); + for (let hop = 0; hop <= maxRedirects; hop++) { + const hostname = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + (isIP(hostname) && !isPublicAddress(hostname)) + ) { + throw new ParamsError(errorMessage); + } + const response = await fetch(url, { dispatcher, redirect: 'manual' }); + const location = response.headers.get('location'); + if (!redirects.has(response.status) || !location) { + return response; + } + await response.body?.cancel(); + if (hop === maxRedirects || location.length > 8192) { + throw new ParamsError(errorMessage); + } + url = new URL(location, url); + } + } catch { + // Do not expose the input, DNS answers, or undici's chained connection error. + throw new ParamsError(errorMessage); + } + throw new ParamsError(errorMessage); +} diff --git a/packages/utils/src/scrape-weblink.ts b/packages/utils/src/scrape-weblink.ts index 71cbac00bc..0e21cf11dc 100644 --- a/packages/utils/src/scrape-weblink.ts +++ b/packages/utils/src/scrape-weblink.ts @@ -1,8 +1,9 @@ import { safeParseURL } from './url'; import { load } from 'cheerio'; +import { safeFetch } from './safe-fetch'; export const scrapeWeblink = async (url: string) => { - const res = await fetch(url); + const res = await safeFetch(url); const html = await res.text(); const $ = load(html); diff --git a/packages/utils/src/server.ts b/packages/utils/src/server.ts new file mode 100644 index 0000000000..600793f38a --- /dev/null +++ b/packages/utils/src/server.ts @@ -0,0 +1,3 @@ +// Keep Node networking dependencies out of the browser-facing entry point. +export * from './safe-fetch'; +export * from './scrape-weblink'; diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index 9954322a10..3c470fffdc 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -34,5 +34,9 @@ }, "include": ["src/**/*"], "exclude": ["node_modules", "dist"], - "references": [{ "path": "../openapi-schema" }, { "path": "../common-types" }] + "references": [ + { "path": "../openapi-schema" }, + { "path": "../common-types" }, + { "path": "../errors" } + ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1cbc4c792a..65a276b4de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1403,6 +1403,9 @@ importers: '@refly/common-types': specifier: workspace:* version: link:../common-types + '@refly/errors': + specifier: workspace:* + version: link:../errors '@refly/openapi-schema': specifier: workspace:* version: link:../openapi-schema @@ -1418,6 +1421,9 @@ importers: deepmerge: specifier: ^4.3.1 version: 4.3.1 + ipaddr.js: + specifier: 2.2.0 + version: 2.2.0 kitoken: specifier: ^0.10.1 version: 0.10.1 @@ -1445,6 +1451,9 @@ importers: tlds: specifier: ^1.255.0 version: 1.259.0 + undici: + specifier: 6.29.0 + version: 6.29.0 uuid: specifier: ^9.0.0 version: 9.0.1 @@ -17760,6 +17769,10 @@ packages: resolution: {integrity: sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==} engines: {node: '>=18.17'} + undici@6.29.0: + resolution: {integrity: sha512-R+RODBqp6i2pPflGdq+xIOUkl+RNfGgHwoinecKu/JCuf2uO06cOKoDbI2P7Dn6KcswdKwrczbU6IYJ6K8X+wg==} + engines: {node: '>=18.17'} + unicode-canonical-property-names-ecmascript@2.0.1: resolution: {integrity: sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==} engines: {node: '>=4'} @@ -39467,6 +39480,8 @@ snapshots: undici@6.21.3: {} + undici@6.29.0: {} + unicode-canonical-property-names-ecmascript@2.0.1: {} unicode-match-property-ecmascript@2.0.0: From eceda89456ab8d9c7414e9fa80f3de3ca260317f Mon Sep 17 00:00:00 2001 From: KevinAndrewDong <2970752254@qq.com> Date: Thu, 1 Oct 2026 15:17:02 +0800 Subject: [PATCH 2/3] fix(utils): preserve safe fetch error and URL compatibility --- .../modules/misc/misc.service.ssrf.spec.ts | 31 +++-- packages/utils/src/safe-fetch.test.ts | 112 ++++++++++++++++-- packages/utils/src/safe-fetch.ts | 45 ++++--- 3 files changed, 151 insertions(+), 37 deletions(-) diff --git a/apps/api/src/modules/misc/misc.service.ssrf.spec.ts b/apps/api/src/modules/misc/misc.service.ssrf.spec.ts index 493a7df821..16cce89b4f 100644 --- a/apps/api/src/modules/misc/misc.service.ssrf.spec.ts +++ b/apps/api/src/modules/misc/misc.service.ssrf.spec.ts @@ -1,4 +1,4 @@ -import { ParamsError } from '@refly/errors'; +import { ConnectionError, ParamsError } from '@refly/errors'; import { safeFetch, scrapeWeblink } from '@refly/utils/server'; import { MiscService } from './misc.service'; @@ -39,19 +39,30 @@ describe('MiscService untrusted downloads', () => { expect(guardedScrape).toHaveBeenCalledWith('https://public.example'); }); - it('propagates the guard rejection in both paths without uploading', async () => { - const blocked = new ParamsError('URL must resolve to a public HTTP(S) destination'); + it.each([ + { + name: 'policy rejection', + error: new ParamsError('URL must resolve to a public HTTP(S) destination'), + url: 'http://127.0.0.1', + }, + { + name: 'connection failure', + error: new ConnectionError('Cannot connect to the URL destination'), + url: 'https://public.example', + }, + ])('propagates $name in both paths without uploading', async ({ error: blocked, url }) => { guardedScrape.mockRejectedValue(blocked); guardedFetch.mockRejectedValue(blocked); const upload = jest.spyOn(service, 'uploadFile'); - await expect(service.scrapeWeblink({ url: 'http://127.0.0.1' })).rejects.toBe(blocked); - await expect( - service.dumpFileFromURL({ uid: 'user' }, { url: 'http://127.0.0.1' }), - ).rejects.toBe(blocked); + await expect(service.scrapeWeblink({ url })).rejects.toBe(blocked); + await expect(service.dumpFileFromURL({ uid: 'user' }, { url })).rejects.toBe(blocked); expect(upload).not.toHaveBeenCalled(); }); - it('keeps binary bytes, filename, MIME type and upload context', async () => { + it.each([ + 'https://public.example/image?token=removed', + `https://public.example/image?signature=${'signed-token'.repeat(1000)}`, + ])('keeps binary bytes, filename, MIME type and upload context for %s', async (url) => { const bytes = new Uint8Array([0, 255, 1, 128]); // Only the Response methods used by this consumer are needed here. guardedFetch.mockResolvedValue({ @@ -64,13 +75,13 @@ describe('MiscService untrusted downloads', () => { await service.dumpFileFromURL( { uid: 'user' }, { - url: 'https://public.example/image?token=removed', + url, entityId: 'resource', entityType: 'resource', }, ), ).toEqual(uploaded); - expect(guardedFetch).toHaveBeenCalledWith('https://public.example/image?token=removed'); + expect(guardedFetch).toHaveBeenCalledWith(url); expect(upload).toHaveBeenCalledWith( { uid: 'user' }, { diff --git a/packages/utils/src/safe-fetch.test.ts b/packages/utils/src/safe-fetch.test.ts index 2e9dd5972e..b25a746922 100644 --- a/packages/utils/src/safe-fetch.test.ts +++ b/packages/utils/src/safe-fetch.test.ts @@ -2,7 +2,7 @@ import { createServer } from 'node:http'; import net, { type LookupFunction, type TcpNetConnectOpts } from 'node:net'; import type { LookupAddress, LookupAllOptions } from 'node:dns'; import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ParamsError } from '@refly/errors'; +import { ConnectionError, ParamsError } from '@refly/errors'; import ipaddr from 'ipaddr.js'; import { Agent, Response, fetch } from 'undici'; import { safeFetch } from './safe-fetch'; @@ -59,6 +59,23 @@ async function expectBlocked(input: string) { } } +async function expectConnectionFailure(input: string) { + try { + await safeFetch(input); + expect.fail('Expected a connection failure'); + } catch (error) { + expect(error).toBeInstanceOf(ConnectionError); + const failed = error as ConnectionError & { cause?: unknown }; + expect(failed.code).toBe('E0001'); + expect(failed.message).toBe('Cannot connect to the URL destination'); + expect(failed.cause).toBeUndefined(); + const details = `${failed.message}\n${failed.stack}\n${JSON.stringify(failed)}`; + for (const secret of ['PRIVATE_SENTINEL', input, '93.184.216.34']) { + expect(details).not.toContain(secret); + } + } +} + beforeEach(() => { network.lookup.mockReset().mockImplementation((_hostname, options, callback) => { expect(options).toEqual({ all: true, verbatim: true }); @@ -84,7 +101,10 @@ describe('safeFetch destination policy', () => { 'https://[::ffff:93.184.216.34]', 'http://public.example', 'https://public.example', - ])('allows public HTTP(S): %s', async (url) => { + 'https://127.0.0.1@public.example', + 'https://PRIVATE_SENTINEL:password@public.example', + `https://public.example/image?signature=${'PRIVATE_SENTINEL'.repeat(1000)}`, + ])('passes public HTTP(S) destinations to fetch: %s', async (url) => { expect(await (await safeFetch(url)).text()).toBe('public content'); }); @@ -142,14 +162,13 @@ describe('safeFetch destination policy', () => { 'http://[::1', 'http://%ff/PRIVATE_SENTINEL', 'http://[fe80::1%25lo0]', - 'https://PRIVATE_SENTINEL:password@public.example', - `https://public.example/${'PRIVATE_SENTINEL'.repeat(1000)}`, + 'https://public.example@127.0.0.1/PRIVATE_SENTINEL', null, undefined, 123, {}, ['https://public.example'], - ])('rejects invalid syntax, encodings, types and oversized URLs: %s', async (input) => { + ])('rejects invalid syntax, encodings and types: %s', async (input) => { await expectBlocked(input as string); expect(network.fetch).not.toHaveBeenCalled(); }); @@ -211,19 +230,61 @@ describe('safeFetch destination policy', () => { expect(network.lookup).toHaveBeenCalledTimes(1); }); - it('normalizes DNS and connection failures without unsafe causes', async () => { + it('normalizes DNS failures as connection errors without unsafe causes', async () => { network.lookup.mockImplementation((_host, _options, callback) => { - callback(new Error('PRIVATE_SENTINEL DNS failure'), []); + callback(Object.assign(new Error('PRIVATE_SENTINEL DNS failure'), { code: 'ENOTFOUND' }), []); }); - await expectBlocked('https://public.example'); + const original = await vi.importActual('undici'); + network.fetch.mockImplementation(original.fetch); + await expectConnectionFailure('https://public.example/PRIVATE_SENTINEL'); + }); + + it.each(['ECONNRESET', 'CERT_HAS_EXPIRED', 'UND_ERR_CONNECT_TIMEOUT'])( + 'normalizes %s after safe DNS validation without unsafe causes', + async (code) => { + // Exercise real undici error wrapping after checked lookup, without dialing. + const dial = vi.spyOn(net, 'connect').mockImplementation(((options: TcpNetConnectOpts) => { + const socket = new net.Socket(); + expect(options.lookup).toBe(connection.lookup); + options.lookup!(options.host!, { all: true }, (error, addresses) => { + expect(error).toBeNull(); + expect(addresses).toEqual([{ address: '93.184.216.34', family: 4 }]); + socket.destroy( + Object.assign(new Error(`PRIVATE_SENTINEL ${code}`), { + code, + cause: new Error('93.184.216.34 PRIVATE_SENTINEL'), + }), + ); + }); + return socket; + }) as typeof net.connect); + const original = await vi.importActual('undici'); + network.fetch.mockImplementation(original.fetch); + try { + await expectConnectionFailure('http://public.example/PRIVATE_SENTINEL'); + expect(network.lookup).toHaveBeenCalledOnce(); + } finally { + dial.mockRestore(); + } + }, + ); + + it('preserves a policy rejection wrapped by undici', async () => { network.fetch.mockRejectedValue( Object.assign(new Error('PRIVATE_SENTINEL'), { - cause: new Error('PRIVATE_SENTINEL resolved address'), + cause: new ParamsError('PRIVATE_SENTINEL resolved address'), }), ); await expectBlocked('https://public.example'); }); + it('keeps the underlying fetch credential restriction without leaking userinfo', async () => { + const original = await vi.importActual('undici'); + network.fetch.mockImplementation(original.fetch); + await expectConnectionFailure('https://PRIVATE_SENTINEL:password@public.example'); + expect(network.lookup).not.toHaveBeenCalled(); + }); + it('normalizes IP parser failures inside the socket lookup callback', async () => { const parse = vi.spyOn(ipaddr, 'process').mockImplementation(() => { throw new Error('PRIVATE_SENTINEL'); @@ -256,7 +317,6 @@ describe('redirects', () => { 'http://[::ffff:127.0.0.1]/PRIVATE_SENTINEL', 'file:///PRIVATE_SENTINEL', 'http://[PRIVATE_SENTINEL', - `https://public.example/${'PRIVATE_SENTINEL'.repeat(1000)}`, ])('blocks unsafe redirects before the next request: %s', async (location) => { network.fetch.mockResolvedValueOnce(new Response(null, { status: 302, headers: { location } })); await expectBlocked('https://public.example'); @@ -290,7 +350,35 @@ describe('redirects', () => { expect(network.fetch).toHaveBeenCalledTimes(2); }); - it('caps a redirect loop at five redirects', async () => { + it('preserves long signed redirect URLs', async () => { + const location = `https://public.example/image?signature=${'PRIVATE_SENTINEL'.repeat(1000)}`; + network.fetch.mockResolvedValueOnce(new Response(null, { status: 302, headers: { location } })); + expect(await (await safeFetch('https://public.example')).text()).toBe('public content'); + expect(String(network.fetch.mock.calls[1][0])).toBe(location); + }); + + it('normalizes a redirect body cancellation failure as a connection error', async () => { + const redirect = new Response('body', { status: 302, headers: { location: '/next' } }); + vi.spyOn(redirect.body!, 'cancel').mockRejectedValue(new Error('PRIVATE_SENTINEL')); + network.fetch.mockResolvedValueOnce(redirect); + await expectConnectionFailure('https://public.example/PRIVATE_SENTINEL'); + expect(network.fetch).toHaveBeenCalledOnce(); + }); + + it('allows twenty public redirects and validates every hop', async () => { + let requests = 0; + network.fetch.mockImplementation(async (input) => { + await resolveAtSocket(new URL(String(input)).hostname); + return requests++ < 20 + ? new Response(null, { status: 302, headers: { location: '/next' } }) + : new Response('public content'); + }); + expect(await (await safeFetch('https://public.example')).text()).toBe('public content'); + expect(network.fetch).toHaveBeenCalledTimes(21); + expect(network.lookup).toHaveBeenCalledTimes(21); + }); + + it('caps a redirect loop at twenty redirects', async () => { network.fetch.mockImplementation( async () => new Response(null, { @@ -299,7 +387,7 @@ describe('redirects', () => { }), ); await expectBlocked('https://public.example/loop'); - expect(network.fetch).toHaveBeenCalledTimes(6); + expect(network.fetch).toHaveBeenCalledTimes(21); }); it('preserves a non-redirect response and a redirect without Location', async () => { diff --git a/packages/utils/src/safe-fetch.ts b/packages/utils/src/safe-fetch.ts index 34a19485b0..3c9a983a27 100644 --- a/packages/utils/src/safe-fetch.ts +++ b/packages/utils/src/safe-fetch.ts @@ -1,13 +1,14 @@ import { lookup } from 'node:dns'; import { isIP } from 'node:net'; -import { ParamsError } from '@refly/errors'; +import { ConnectionError, ParamsError } from '@refly/errors'; import ipaddr from 'ipaddr.js'; import { Agent, fetch } from 'undici'; const publicIPv6 = ipaddr.IPv6.parse('2000::'); const redirects = new Set([301, 302, 303, 307, 308]); -const maxRedirects = 5; +const maxRedirects = 20; const errorMessage = 'URL must resolve to a public HTTP(S) destination'; +const connectionErrorMessage = 'Cannot connect to the URL destination'; function isPublicAddress(address: string): boolean { try { @@ -32,11 +33,11 @@ const dispatcher = new Agent({ // This is the socket's lookup: only checked addresses reach connect(). // Check all answers so a mixed public/private DNS response fails closed. lookup(hostname, { all: true, verbatim: true }, (error, addresses) => { - if ( - error || - !addresses?.length || - addresses.some(({ address }) => !isPublicAddress(address)) - ) { + if (error) { + callback(new ConnectionError(connectionErrorMessage), '', 0); + return; + } + if (!addresses?.length || addresses.some(({ address }) => !isPublicAddress(address))) { callback(new ParamsError(errorMessage), '', 0); return; } @@ -53,8 +54,8 @@ const dispatcher = new Agent({ /** Server-only GET for untrusted URLs, including every redirect destination. */ export async function safeFetch(input: string) { try { - // Reject coercion of non-string input and bound URL parsing work. - if (typeof input !== 'string' || input.length > 8192) { + // Reject coercion of non-string input without restricting signed URL length. + if (typeof input !== 'string') { throw new ParamsError(errorMessage); } let url = new URL(input); @@ -62,25 +63,39 @@ export async function safeFetch(input: string) { const hostname = url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname; if ( !['http:', 'https:'].includes(url.protocol) || - url.username || - url.password || (isIP(hostname) && !isPublicAddress(hostname)) ) { throw new ParamsError(errorMessage); } - const response = await fetch(url, { dispatcher, redirect: 'manual' }); + const response = await fetch(url, { dispatcher, redirect: 'manual' }).catch( + (error: unknown) => { + // Undici wraps socket lookup policy rejections in a fetch error's cause. + if ( + error instanceof ParamsError || + (error instanceof Error && 'cause' in error && error.cause instanceof ParamsError) + ) { + throw new ParamsError(errorMessage); + } + throw new ConnectionError(connectionErrorMessage); + }, + ); const location = response.headers.get('location'); if (!redirects.has(response.status) || !location) { return response; } - await response.body?.cancel(); - if (hop === maxRedirects || location.length > 8192) { + await response.body?.cancel().catch(() => { + throw new ConnectionError(connectionErrorMessage); + }); + if (hop === maxRedirects) { throw new ParamsError(errorMessage); } url = new URL(location, url); } - } catch { + } catch (error) { // Do not expose the input, DNS answers, or undici's chained connection error. + if (error instanceof ConnectionError) { + throw new ConnectionError(connectionErrorMessage); + } throw new ParamsError(errorMessage); } throw new ParamsError(errorMessage); From 721df21872bf6e40d4918e007c71612347ad40a5 Mon Sep 17 00:00:00 2001 From: KevinAndrewDong <2970752254@qq.com> Date: Thu, 8 Oct 2026 03:25:14 +0800 Subject: [PATCH 3/3] test(utils): exercise guarded HTTPS fetching --- .../src/__fixtures__/safe-fetch-tls/README.md | 13 + .../src/__fixtures__/safe-fetch-tls/ca.crt | 11 + .../__fixtures__/safe-fetch-tls/server.crt | 13 + .../__fixtures__/safe-fetch-tls/server.key | 5 + .../__fixtures__/safe-fetch-tls/untrusted.crt | 12 + .../__fixtures__/safe-fetch-tls/untrusted.key | 5 + packages/utils/src/safe-fetch.https.test.ts | 263 ++++++++++++++++++ 7 files changed, 322 insertions(+) create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/README.md create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/ca.crt create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/server.crt create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/server.key create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.crt create mode 100644 packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.key create mode 100644 packages/utils/src/safe-fetch.https.test.ts diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/README.md b/packages/utils/src/__fixtures__/safe-fetch-tls/README.md new file mode 100644 index 0000000000..5f0f618bc8 --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/README.md @@ -0,0 +1,13 @@ +# HTTPS test certificates + +These static P-256 certificates and openly published private keys are test fixtures only. +They expire in September 2126 and require no certificate tooling when tests run. + +`server.crt` is signed by `ca.crt`, with DNS SANs for `public.fixture.example` and +`redirect.fixture.example`. `untrusted.crt` is self-signed with a matching SAN for +`public.fixture.example`; the test Agent trusts only `ca.crt`. + +The HTTPS tests retain the original URL hostname, SNI and default TLS validation. +Their routing wrapper invokes the production socket lookup before replacing an +approved public DNS answer with the local fixture address. Rejected answers never +reach the local server. These fixtures use no public network or TLS bypass. diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/ca.crt b/packages/utils/src/__fixtures__/safe-fetch-tls/ca.crt new file mode 100644 index 0000000000..d6574463d2 --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/ca.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlDCCATmgAwIBAgIUNi2AMFDQkMctYipPInmQcwJHv20wCgYIKoZIzj0EAwIw +HjEcMBoGA1UEAwwTUmVmbHkgSFRUUFMgdGVzdCBDQTAgFw0yNjEwMDcxOTEwMjla +GA8yMTI2MDkxMzE5MTAyOVowHjEcMBoGA1UEAwwTUmVmbHkgSFRUUFMgdGVzdCBD +QTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABIS0VRadGkdjlihkSy0VLx8ftcmT +FBBKpb+vxXuT2W7+V2IKYvtWfTgf674Tj+qkC4iPib1M54PyTy49n/MfHIWjUzBR +MB0GA1UdDgQWBBRjYbHjR8FCwCVBjhZGnwINiT6hGDAfBgNVHSMEGDAWgBRjYbHj +R8FCwCVBjhZGnwINiT6hGDAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0kA +MEYCIQChXwuPKnO5SbZcXLffIpR3kvf9kYSS5p1udNwt+7brIAIhAPxtGlcVLlpO +CIdpZxz9Xyit8wa1LT+LyzSycqhE62V0 +-----END CERTIFICATE----- diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/server.crt b/packages/utils/src/__fixtures__/safe-fetch-tls/server.crt new file mode 100644 index 0000000000..1b048d08d8 --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/server.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB4zCCAYqgAwIBAgIBAjAKBggqhkjOPQQDAjAeMRwwGgYDVQQDDBNSZWZseSBI +VFRQUyB0ZXN0IENBMCAXDTI2MTAwNzE5MTAyOVoYDzIxMjYwOTEzMTkxMDI5WjAh +MR8wHQYDVQQDDBZwdWJsaWMuZml4dHVyZS5leGFtcGxlMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEz3tmXqIAbJ+VB8ngv9Ih4gh8QXzY7nicS6cjVnV+DasSKmNa +4SLi7UyWhkCHiKmKi234ILBHimRr0puKfieROqOBszCBsDA7BgNVHREENDAyghZw +dWJsaWMuZml4dHVyZS5leGFtcGxlghhyZWRpcmVjdC5maXh0dXJlLmV4YW1wbGUw +DAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUH +AwEwHQYDVR0OBBYEFIzEVVuINcCCFGLun0zKQIP/u2ClMB8GA1UdIwQYMBaAFGNh +seNHwULAJUGOFkafAg2JPqEYMAoGCCqGSM49BAMCA0cAMEQCID3jApZcQjCXr7W2 +lW8o1Z76XSDcV5fSAjKUUgjCqbOMAiB9es655PLg1PyHUWK4QTIAfRQST4JU2cST +pvtdtrBEQw== +-----END CERTIFICATE----- diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/server.key b/packages/utils/src/__fixtures__/safe-fetch-tls/server.key new file mode 100644 index 0000000000..61212e4517 --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/server.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgH3GY8T8hJgeusffg +c+RuiDUFQ68Hm0CruAb+6qo28d2hRANCAATPe2ZeogBsn5UHyeC/0iHiCHxBfNju +eJxLpyNWdX4NqxIqY1rhIuLtTJaGQIeIqYqLbfggsEeKZGvSm4p+J5E6 +-----END PRIVATE KEY----- diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.crt b/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.crt new file mode 100644 index 0000000000..700a2e30ea --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.crt @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBvTCCAWKgAwIBAgIUBElbBfHezSeR+F4VRBOd3twyC3owCgYIKoZIzj0EAwIw +ITEfMB0GA1UEAwwWcHVibGljLmZpeHR1cmUuZXhhbXBsZTAgFw0yNjEwMDcxOTEw +MjlaGA8yMTI2MDkxMzE5MTAyOVowITEfMB0GA1UEAwwWcHVibGljLmZpeHR1cmUu +ZXhhbXBsZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGFNDO+FSvubyo6+byke +NsDoqr4H9IUoEgoSDfgu7LigEBsKo2ByzHBK3Lso7y9Ayhg6+z4KqLgBXGPDckPT +K12jdjB0MB0GA1UdDgQWBBRoAbjhOyu8QfHIFpLsT1MlIvwtazAfBgNVHSMEGDAW +gBRoAbjhOyu8QfHIFpLsT1MlIvwtazAPBgNVHRMBAf8EBTADAQH/MCEGA1UdEQQa +MBiCFnB1YmxpYy5maXh0dXJlLmV4YW1wbGUwCgYIKoZIzj0EAwIDSQAwRgIhAPLk +yFnNPDhDik/WJcVsBkcPf5/H1yWhewKT5r20kZ1lAiEAyJ30EeAjAdDN6KYJiuZC +UTssap5Sgkg/lzUSLl6F8uI= +-----END CERTIFICATE----- diff --git a/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.key b/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.key new file mode 100644 index 0000000000..640ab71ac7 --- /dev/null +++ b/packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg5JK7RK6ltRdqMSPC +mMndY23YYX8qTKy1E1R8vQ1SsbWhRANCAARhTQzvhUr7m8qOvm8pHjbA6Kq+B/SF +KBIKEg34Luy4oBAbCqNgcsxwSty7KO8vQMoYOvs+Cqi4AVxjw3JD0ytd +-----END PRIVATE KEY----- diff --git a/packages/utils/src/safe-fetch.https.test.ts b/packages/utils/src/safe-fetch.https.test.ts new file mode 100644 index 0000000000..c2de7885db --- /dev/null +++ b/packages/utils/src/safe-fetch.https.test.ts @@ -0,0 +1,263 @@ +import type { LookupAddress, LookupAllOptions } from 'node:dns'; +import { readFileSync } from 'node:fs'; +import type { RequestListener } from 'node:http'; +import { createServer } from 'node:https'; +import type { AddressInfo, LookupFunction, TcpNetConnectOpts } from 'node:net'; +import { join } from 'node:path'; +import type { Duplex } from 'node:stream'; +import tls from 'node:tls'; +import type { ConnectionOptions } from 'node:tls'; +import { ConnectionError, ParamsError } from '@refly/errors'; +import type { Agent } from 'undici'; +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { safeFetch } from './safe-fetch'; + +type Lookup = ( + hostname: string, + options: LookupAllOptions, + callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void, +) => void; +const network = vi.hoisted(() => ({ lookup: vi.fn() })); +const connection = vi.hoisted(() => ({ + lookup: undefined as unknown as LookupFunction, + agent: undefined as unknown as Agent, +})); +vi.mock('node:dns', () => ({ lookup: network.lookup })); +vi.mock('undici', async (importOriginal) => { + const original = await importOriginal(); + const { readFileSync } = await import('node:fs'); + const { join } = await import('node:path'); + return { + ...original, + Agent: class extends original.Agent { + constructor(options: Agent.Options) { + expect(typeof options.connect).toBe('object'); + const connect = options.connect as ConnectionOptions; + connection.lookup = connect.lookup!; + // Trust only the fixture CA; retain Node's certificate and hostname checks. + super({ + ...options, + connect: { + ...connect, + ca: readFileSync(join(__dirname, '__fixtures__/safe-fetch-tls/ca.crt')), + }, + }); + connection.agent = this; + } + }, + }; +}); + +beforeEach(() => { + network.lookup.mockReset().mockImplementation((_hostname, options, callback) => { + expect(options).toEqual({ all: true, verbatim: true }); + callback(null, [{ address: '93.184.216.34', family: 4 }]); + }); +}); +afterAll(() => connection.agent.close()); + +async function withHttpsFixture( + certificate: 'server' | 'untrusted', + check: (fixture: { + port: number; + approved: string[]; + sni: string[]; + requests: string[]; + tlsErrors: string[]; + }) => Promise, + handler: RequestListener = (_request, response) => response.end('secure content'), +) { + const fixturePath = join(__dirname, '__fixtures__/safe-fetch-tls'); + const approved: string[] = []; + const sni: string[] = []; + const requests: string[] = []; + const tlsErrors: string[] = []; + const sockets = new Set(); + const server = createServer( + { + cert: readFileSync(join(fixturePath, `${certificate}.crt`)), + key: readFileSync(join(fixturePath, `${certificate}.key`)), + }, + (request, response) => { + requests.push(`${request.headers.host}${request.url}`); + handler(request, response); + }, + ); + server.on('connection', (socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + server.on('secureConnection', (socket) => + sni.push((socket as tls.TLSSocket & { servername: string }).servername), + ); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const port = (server.address() as AddressInfo).port; + const connect = tls.connect; + // Intercept only routing after the production socket lookup approves DNS. + // The real TLS handshake still uses the URL hostname for SNI and verification. + const dial = vi.spyOn(tls, 'connect').mockImplementation((( + options: ConnectionOptions & Pick, + ) => { + expect(options.lookup).toBe(connection.lookup); + expect(options.autoSelectFamily).toBe(true); + expect(options.servername).toBe(options.host); + expect(options.rejectUnauthorized).not.toBe(false); + expect(options.checkServerIdentity).toBeUndefined(); + const socket = connect({ + ...options, + lookup(hostname, lookupOptions, callback) { + connection.lookup(hostname, lookupOptions, (error, addresses) => { + if (error) { + callback(error, '', 0); + return; + } + expect(lookupOptions.all).toBe(true); + expect(addresses).toEqual([{ address: '93.184.216.34', family: 4 }]); + approved.push(hostname); + callback(null, [{ address: '127.0.0.1', family: 4 }]); + }); + }, + }); + socket.on('error', (error: NodeJS.ErrnoException) => tlsErrors.push(error.code!)); + return socket; + }) as typeof tls.connect); + try { + await check({ port, approved, sni, requests, tlsErrors }); + } finally { + dial.mockRestore(); + for (const socket of sockets) socket.destroy(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +} + +async function expectSanitizedFailure(input: string, blocked = false) { + try { + await safeFetch(input); + expect.fail('Expected the HTTPS destination to be rejected'); + } catch (error) { + expect(error).toBeInstanceOf(blocked ? ParamsError : ConnectionError); + const failure = error as ConnectionError & { cause?: unknown }; + expect(failure.code).toBe(blocked ? 'E0003' : 'E0001'); + expect(failure.message).toBe( + blocked + ? 'URL must resolve to a public HTTP(S) destination' + : 'Cannot connect to the URL destination', + ); + expect(failure.cause).toBeUndefined(); + const details = `${failure.message}\n${failure.stack}\n${JSON.stringify(failure)}`; + for (const secret of [ + 'PRIVATE_SENTINEL', + input, + new URL(input).hostname, + '93.184.216.34', + '127.0.0.1', + '10.0.0.1', + 'Refly HTTPS test CA', + ]) { + expect(details).not.toContain(secret); + } + } +} + +describe('safeFetch real HTTPS connections', () => { + it('preserves the hostname, SNI and checked DNS through a trusted TLS handshake', async () => { + await withHttpsFixture('server', async ({ port, approved, sni, requests }) => { + const url = `https://public.fixture.example:${port}/content`; + expect(await (await safeFetch(url)).text()).toBe('secure content'); + expect(sni).toEqual(['public.fixture.example']); + expect(requests).toEqual([`public.fixture.example:${port}/content`]); + expect(approved).toEqual(['public.fixture.example']); + expect(network.lookup.mock.calls.map(([hostname]) => hostname)).toEqual(approved); + }); + }); + + it.each([ + ['server', 'wrong.fixture.example'], + ['untrusted', 'public.fixture.example'], + ] as const)('rejects %s certificate for %s without exposing TLS details', async (cert, host) => { + await withHttpsFixture(cert, async ({ port, approved, requests, tlsErrors }) => { + await expectSanitizedFailure(`https://${host}:${port}/PRIVATE_SENTINEL`); + expect(approved).toEqual([host]); + expect(network.lookup.mock.calls.map(([hostname]) => hostname)).toEqual([host]); + expect(requests).toEqual([]); + expect(tlsErrors).toEqual([ + cert === 'server' ? 'ERR_TLS_CERT_ALTNAME_INVALID' : 'DEPTH_ZERO_SELF_SIGNED_CERT', + ]); + }); + }); + + it('revalidates public HTTPS redirects with fresh DNS and TLS hostname checks', async () => { + await withHttpsFixture( + 'server', + async ({ port, approved, sni, requests }) => { + const url = `https://public.fixture.example:${port}/start`; + expect(await (await safeFetch(url)).text()).toBe('redirect content'); + expect(approved).toEqual(['public.fixture.example', 'redirect.fixture.example']); + expect(network.lookup.mock.calls.map(([hostname]) => hostname)).toEqual(approved); + expect(sni).toEqual(approved); + expect(requests).toEqual([ + `public.fixture.example:${port}/start`, + `redirect.fixture.example:${port}/final`, + ]); + }, + (request, response) => { + if (request.url === '/start') { + response.writeHead(302, { + location: `https://redirect.fixture.example:${request.socket.localPort}/final`, + }); + } else { + response.write('redirect content'); + } + response.end(); + }, + ); + }); + + it.each(['private IP', 'private DNS', 'DNS rebinding'])( + 'blocks an HTTPS redirect to %s before another local connection', + async (destination) => { + network.lookup.mockImplementation((_hostname, options, callback) => { + expect(options).toEqual({ all: true, verbatim: true }); + callback(null, [ + { + address: network.lookup.mock.calls.length === 1 ? '93.184.216.34' : '10.0.0.1', + family: 4, + }, + ]); + }); + await withHttpsFixture( + 'server', + async ({ port, approved, sni, requests }) => { + await expectSanitizedFailure( + `https://public.fixture.example:${port}/PRIVATE_SENTINEL`, + true, + ); + expect(approved).toEqual(['public.fixture.example']); + expect(sni).toEqual(approved); + expect(requests).toEqual([`public.fixture.example:${port}/PRIVATE_SENTINEL`]); + const expected = ['public.fixture.example']; + if (destination !== 'private IP') { + expected.push(destination === 'private DNS' ? 'internal.fixture.example' : expected[0]); + } + expect(network.lookup.mock.calls.map(([hostname]) => hostname)).toEqual(expected); + }, + (request, response) => { + const hostname = + destination === 'private IP' + ? '127.0.0.1' + : destination === 'private DNS' + ? 'internal.fixture.example' + : 'public.fixture.example'; + response.writeHead(302, { + location: `https://${hostname}:${request.socket.localPort}/PRIVATE_SENTINEL`, + }); + response.end(); + }, + ); + }, + ); +});