fix(utils): prevent SSRF in untrusted URL downloads

apps/api/src/modules/misc/misc.service.ssrf.spec.ts

@@ -0,0 +1,95 @@
+import { ConnectionError, ParamsError } from '@refly/errors';
+import { safeFetch, scrapeWeblink } from '@refly/utils/server';
+import { MiscService } from './misc.service';
+
+jest.mock('@refly/utils/server', () => ({ safeFetch: jest.fn(), scrapeWeblink: jest.fn() }));
+jest.mock('@refly/utils', () => ({}));
+jest.mock('@paralleldrive/cuid2', () => ({ createId: jest.fn() }));
+jest.mock('sharp', () => jest.fn());
+jest.mock('../common/prisma.service', () => ({ PrismaService: class {} }));
+jest.mock('../common/redis.service', () => ({ RedisService: class {} }));
+jest.mock('../common/object-storage', () => ({
+  ObjectStorageService: class {},
+  OSS_EXTERNAL: 'external',
+  OSS_INTERNAL: 'internal',
+}));
+jest.mock('../knowledge/parsers/pandoc.parser', () => ({ PandocParser: class {} }));
+jest.mock('../../utils', () => ({
+  QUEUE_IMAGE_PROCESSING: 'images',
+  QUEUE_CLEAN_STATIC_FILES: 'cleanup',
+}));
+
+describe('MiscService untrusted downloads', () => {
+  const guardedFetch = jest.mocked(safeFetch);
+  const guardedScrape = jest.mocked(scrapeWeblink);
+  const directFetch = jest.spyOn(globalThis, 'fetch');
+  let service: MiscService;
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+    directFetch.mockRejectedValue(new Error('Direct network access is forbidden in this test'));
+    service = Object.create(MiscService.prototype) as MiscService;
+  });
+  afterAll(() => directFetch.mockRestore());
+
+  it('routes metadata scraping through the guarded scraper', async () => {
+    const result = { title: 'Public title', description: 'Public description', image: 'image.png' };
+    guardedScrape.mockResolvedValue(result);
+    expect(await service.scrapeWeblink({ url: 'https://public.example' })).toEqual(result);
+    expect(guardedScrape).toHaveBeenCalledWith('https://public.example');
+  });
+
+  it.each([
+    {
+      name: 'policy rejection',
+      error: new ParamsError('URL must resolve to a public HTTP(S) destination'),
+      url: 'http://127.0.0.1',
+    },
+    {
+      name: 'connection failure',
+      error: new ConnectionError('Cannot connect to the URL destination'),
+      url: 'https://public.example',
+    },
+  ])('propagates $name in both paths without uploading', async ({ error: blocked, url }) => {
+    guardedScrape.mockRejectedValue(blocked);
+    guardedFetch.mockRejectedValue(blocked);
+    const upload = jest.spyOn(service, 'uploadFile');
+    await expect(service.scrapeWeblink({ url })).rejects.toBe(blocked);
+    await expect(service.dumpFileFromURL({ uid: 'user' }, { url })).rejects.toBe(blocked);
+    expect(upload).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    'https://public.example/image?token=removed',
+    `https://public.example/image?signature=${'signed-token'.repeat(1000)}`,
+  ])('keeps binary bytes, filename, MIME type and upload context for %s', async (url) => {
+    const bytes = new Uint8Array([0, 255, 1, 128]);
+    // Only the Response methods used by this consumer are needed here.
+    guardedFetch.mockResolvedValue({
+      arrayBuffer: async () => bytes.buffer,
+      headers: new Headers({ 'content-type': 'image/png' }),
+    } as Awaited<ReturnType<typeof safeFetch>>);
+    const uploaded = { url: 'https://files.example/image.png', storageKey: 'image' };
+    const upload = jest.spyOn(service, 'uploadFile').mockResolvedValue(uploaded);
+    expect(
+      await service.dumpFileFromURL(
+        { uid: 'user' },
+        {
+          url,
+          entityId: 'resource',
+          entityType: 'resource',
+        },
+      ),
+    ).toEqual(uploaded);
+    expect(guardedFetch).toHaveBeenCalledWith(url);
+    expect(upload).toHaveBeenCalledWith(
+      { uid: 'user' },
+      {
+        file: { buffer: Buffer.from(bytes), mimetype: 'image/png', originalname: 'image.png' },
+        entityId: 'resource',
+        entityType: 'resource',
+        visibility: 'private',
+      },
+    );
+  });
+});

apps/api/src/modules/misc/misc.service.ts

@@ -28,11 +28,11 @@ import path from 'node:path';
 import { ConfigService } from '@nestjs/config';
 import {
   omit,
-  scrapeWeblink,
   getSafeMimeType,
   runModuleInitWithTimeoutAndRetry,
   safeParseJSON,
 } from '@refly/utils';
+import { safeFetch, scrapeWeblink } from '@refly/utils/server';
 import { QUEUE_IMAGE_PROCESSING, QUEUE_CLEAN_STATIC_FILES, streamToBuffer } from '../../utils';
 import {
   CanvasNotFoundError,
@@ -174,7 +174,7 @@ export class MiscService implements OnModuleInit {
     },
   ): Promise<UploadResponse['data']> {
     const { url, entityId, entityType, visibility = 'private' } = param;
-    const res = await fetch(url);
+    const res = await safeFetch(url);
     const buffer = await res.arrayBuffer();
 
     // Extract filename from URL, removing query parameters and fragments

packages/utils/package.json

@@ -6,11 +6,13 @@
   "types": "src/index.ts",
   "exports": {
     ".": "./src/index.ts",
-    "./token": "./src/token.ts"
+    "./token": "./src/token.ts",
+    "./server": "./src/server.ts"
   },
   "typesVersions": {
     "*": {
-      "token": ["src/token.ts"]
+      "token": ["src/token.ts"],
+      "server": ["src/server.ts"]
     }
   },
   "scripts": {
@@ -25,11 +27,13 @@
   "dependencies": {
     "@paralleldrive/cuid2": "^2.2.2",
     "@refly/common-types": "workspace:*",
+    "@refly/errors": "workspace:*",
     "@refly/openapi-schema": "workspace:*",
     "@tiptap/pm": "2.7.4",
     "cheerio": "~1.0.0",
     "clsx": "^2.1.0",
     "deepmerge": "^4.3.1",
+    "ipaddr.js": "2.2.0",
     "linkify-it": "^5.0.0",
     "markdown-it": "~14.1.0",
     "md5": "^2.3.0",
@@ -39,6 +43,7 @@
     "tailwind-merge": "^2.2.2",
     "kitoken": "^0.10.1",
     "tlds": "^1.255.0",
+    "undici": "6.29.0",
     "uuid": "^9.0.0",
     "y-prosemirror": "~1.2.6",
     "y-protocols": "~1.0.6",

packages/utils/src/__fixtures__/safe-fetch-tls/README.md

@@ -0,0 +1,13 @@
+# HTTPS test certificates
+
+These static P-256 certificates and openly published private keys are test fixtures only.
+They expire in September 2126 and require no certificate tooling when tests run.
+
+`server.crt` is signed by `ca.crt`, with DNS SANs for `public.fixture.example` and
+`redirect.fixture.example`. `untrusted.crt` is self-signed with a matching SAN for
+`public.fixture.example`; the test Agent trusts only `ca.crt`.
+
+The HTTPS tests retain the original URL hostname, SNI and default TLS validation.
+Their routing wrapper invokes the production socket lookup before replacing an
+approved public DNS answer with the local fixture address. Rejected answers never
+reach the local server. These fixtures use no public network or TLS bypass.

packages/utils/src/__fixtures__/safe-fetch-tls/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBlDCCATmgAwIBAgIUNi2AMFDQkMctYipPInmQcwJHv20wCgYIKoZIzj0EAwIw
+HjEcMBoGA1UEAwwTUmVmbHkgSFRUUFMgdGVzdCBDQTAgFw0yNjEwMDcxOTEwMjla
+GA8yMTI2MDkxMzE5MTAyOVowHjEcMBoGA1UEAwwTUmVmbHkgSFRUUFMgdGVzdCBD
+QTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABIS0VRadGkdjlihkSy0VLx8ftcmT
+FBBKpb+vxXuT2W7+V2IKYvtWfTgf674Tj+qkC4iPib1M54PyTy49n/MfHIWjUzBR
+MB0GA1UdDgQWBBRjYbHjR8FCwCVBjhZGnwINiT6hGDAfBgNVHSMEGDAWgBRjYbHj
+R8FCwCVBjhZGnwINiT6hGDAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0kA
+MEYCIQChXwuPKnO5SbZcXLffIpR3kvf9kYSS5p1udNwt+7brIAIhAPxtGlcVLlpO
+CIdpZxz9Xyit8wa1LT+LyzSycqhE62V0
+-----END CERTIFICATE-----

packages/utils/src/__fixtures__/safe-fetch-tls/server.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB4zCCAYqgAwIBAgIBAjAKBggqhkjOPQQDAjAeMRwwGgYDVQQDDBNSZWZseSBI
+VFRQUyB0ZXN0IENBMCAXDTI2MTAwNzE5MTAyOVoYDzIxMjYwOTEzMTkxMDI5WjAh
+MR8wHQYDVQQDDBZwdWJsaWMuZml4dHVyZS5leGFtcGxlMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEz3tmXqIAbJ+VB8ngv9Ih4gh8QXzY7nicS6cjVnV+DasSKmNa
+4SLi7UyWhkCHiKmKi234ILBHimRr0puKfieROqOBszCBsDA7BgNVHREENDAyghZw
+dWJsaWMuZml4dHVyZS5leGFtcGxlghhyZWRpcmVjdC5maXh0dXJlLmV4YW1wbGUw
+DAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUH
+AwEwHQYDVR0OBBYEFIzEVVuINcCCFGLun0zKQIP/u2ClMB8GA1UdIwQYMBaAFGNh
+seNHwULAJUGOFkafAg2JPqEYMAoGCCqGSM49BAMCA0cAMEQCID3jApZcQjCXr7W2
+lW8o1Z76XSDcV5fSAjKUUgjCqbOMAiB9es655PLg1PyHUWK4QTIAfRQST4JU2cST
+pvtdtrBEQw==
+-----END CERTIFICATE-----

packages/utils/src/__fixtures__/safe-fetch-tls/server.key

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgH3GY8T8hJgeusffg
+c+RuiDUFQ68Hm0CruAb+6qo28d2hRANCAATPe2ZeogBsn5UHyeC/0iHiCHxBfNju
+eJxLpyNWdX4NqxIqY1rhIuLtTJaGQIeIqYqLbfggsEeKZGvSm4p+J5E6
+-----END PRIVATE KEY-----

packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIBvTCCAWKgAwIBAgIUBElbBfHezSeR+F4VRBOd3twyC3owCgYIKoZIzj0EAwIw
+ITEfMB0GA1UEAwwWcHVibGljLmZpeHR1cmUuZXhhbXBsZTAgFw0yNjEwMDcxOTEw
+MjlaGA8yMTI2MDkxMzE5MTAyOVowITEfMB0GA1UEAwwWcHVibGljLmZpeHR1cmUu
+ZXhhbXBsZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGFNDO+FSvubyo6+byke
+NsDoqr4H9IUoEgoSDfgu7LigEBsKo2ByzHBK3Lso7y9Ayhg6+z4KqLgBXGPDckPT
+K12jdjB0MB0GA1UdDgQWBBRoAbjhOyu8QfHIFpLsT1MlIvwtazAfBgNVHSMEGDAW
+gBRoAbjhOyu8QfHIFpLsT1MlIvwtazAPBgNVHRMBAf8EBTADAQH/MCEGA1UdEQQa
+MBiCFnB1YmxpYy5maXh0dXJlLmV4YW1wbGUwCgYIKoZIzj0EAwIDSQAwRgIhAPLk
+yFnNPDhDik/WJcVsBkcPf5/H1yWhewKT5r20kZ1lAiEAyJ30EeAjAdDN6KYJiuZC
+UTssap5Sgkg/lzUSLl6F8uI=
+-----END CERTIFICATE-----

packages/utils/src/__fixtures__/safe-fetch-tls/untrusted.key

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg5JK7RK6ltRdqMSPC
+mMndY23YYX8qTKy1E1R8vQ1SsbWhRANCAARhTQzvhUr7m8qOvm8pHjbA6Kq+B/SF
+KBIKEg34Luy4oBAbCqNgcsxwSty7KO8vQMoYOvs+Cqi4AVxjw3JD0ytd
+-----END PRIVATE KEY-----

packages/utils/src/index.ts

@@ -6,7 +6,6 @@ export * from './parse';
 export * from './editor';
 export * from './typesafe';
 export * from './url';
-export * from './scrape-weblink';
 export * from './models';
 export * from './event-emitter/editor';
 export * from './event-emitter/model';