Add a marketing tool API with marketer keys and audience queries

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+- added: Marketing tool API endpoints: `POST /marketing/test` sends a single test push, `POST /marketing/query` lists the devices a send would reach (filtered by include/exclude lists of cities and regions), and `POST /marketing/push` sends to the device list a query returned.
+- added: A `marketer` flag and a `targetApiKeys` list on API keys. The marketing endpoints require the flag (or `admin`) and only reach devices registered under the listed keys, so the api keys baked into the apps cannot call them.
+- fixed: Recognize Firebase's current "token not registered" error, so devices whose app has been uninstalled get their tokens cleared instead of being retried on every send forever.
+- fixed: Log the device and error when a push fails. The details were being passed to Pino in the wrong argument order, so every failure logged as a bare "Unknown error".
+
 ## 2.5.0 (2025-05-08)
 
 - changed: Move API keys to a separate settings document.

pushServerConfig.sample.json

@@ -0,0 +1,7 @@
+{
+  "listenHost": "127.0.0.1",
+  "listenPort": 8008,
+  "amqpUri": "amqp://username:password@localhost:5672",
+  "couchUri": "http://username:password@localhost:5984",
+  "currentCluster": ""
+}

src/daemons/publishDaemon.ts

@@ -9,6 +9,7 @@ import {
   getDevicesByLoginId
 } from '../db/couchDevices'
 import { DbConnections } from '../db/dbConnections'
+import { isUnregisteredToken } from '../util/firebaseErrors'
 import { logger } from '../util/logger'
 import { asRabbitMessage, SendableMessage } from '../util/pushSender'
 import { runDaemon } from './runDaemon'
@@ -88,12 +89,14 @@ async function sendToDevice(
       data: message.data ?? {}
     })
   } catch (error) {
-    if (String(error).includes('not a valid FCM registration token')) {
+    if (isUnregisteredToken(error)) {
       logger.info(`Disabling device: ${deviceId}`)
       deviceRow.device.deviceToken = undefined
       await deviceRow.save()
     } else {
-      logger.info('Unknown error', { deviceId, error })
+      // Pino takes the details first and the message second. Passing them the
+      // other way around silently drops them:
+      logger.info({ deviceId, error: String(error) }, 'Unknown error')
     }
   }
 }

src/db/couchApiKeys.ts

@@ -1,4 +1,4 @@
-import { asBoolean, asObject, asOptional, asString } from 'cleaners'
+import { asArray, asBoolean, asObject, asOptional, asString } from 'cleaners'
 import {
   asCouchDoc,
   asMaybeNotFoundError,
@@ -32,7 +32,9 @@ export const asCouchApiKey = asCouchDoc(
   asObject<CouchApiKey>({
     appId: asString,
     admin: asBoolean,
-    adminsdk: asOptional(asFirebaseAdminKey)
+    adminsdk: asOptional(asFirebaseAdminKey),
+    marketer: asOptional(asBoolean, false),
+    targetApiKeys: asOptional(asArray(asString), () => [])
   })
 )
 

src/db/couchDevices.ts

@@ -25,6 +25,10 @@ import { asBase64 } from '../types/pushCleaners'
 import { Device } from '../types/pushTypes'
 import { DbConnections } from './dbConnections'
 
+// Couch rejects an `_all_docs` request with no keys, and very large key lists
+// make for unwieldy requests, so batch lookups at this size:
+const FETCH_BATCH_SIZE = 500
+
 /**
  * A device returned from the database.
  * To make changes, edit the `device` object, then call `save`.
@@ -117,12 +121,48 @@ const locationByRegionDesign = makeJsDesign('locationByRegion', ({ emit }) => ({
   reduce: '_count'
 }))
 
+/**
+ * Looks up devices by API key, then by IP location.
+ *
+ * The row value carries everything an audience query needs to decide whether
+ * a device matches and to describe it, so the query never has to fetch the
+ * documents themselves. Devices with no location are absent from this view,
+ * and are therefore unreachable by location targeting.
+ */
+const apiKeyLocationDesign = makeJsDesign('apiKeyLocation', ({ emit }) => ({
+  map: function (doc) {
+    if (doc.apiKey == null || doc.location == null) return
+    emit(
+      [
+        doc.apiKey,
+        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+        doc.location.country || '',
+        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+        doc.location.region || '',
+        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+        doc.location.city || ''
+      ],
+      {
+        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+        region: doc.location.region || '',
+        // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+        city: doc.location.city || '',
+        deviceToken: doc.deviceToken,
+        ignoreMarketing: doc.ignoreMarketing === true,
+        visited: doc.visited
+      }
+    )
+  },
+  reduce: '_count'
+}))
+
 export const couchDevicesSetup: DatabaseSetup = {
   name: 'push-devices',
   documents: {
     '_design/loginId': loginIdDesign,
     '_design/locationByCity': locationByCityDesign,
-    '_design/locationByRegion': locationByRegionDesign
+    '_design/locationByRegion': locationByRegionDesign,
+    '_design/apiKeyLocation': apiKeyLocationDesign
   }
 }
 
@@ -217,6 +257,45 @@ export async function getDeviceById(
   return makeDeviceRow(db, asCouchDevice(raw))
 }
 
+/** One batch of a by-id lookup, along with how far through the ids we are. */
+export interface DeviceBatch {
+  deviceRows: DeviceRow[]
+  /** How many of the requested ids have been read so far. */
+  idsRead: number
+}
+
+/**
+ * Looks up many devices at once, skipping any ids that are missing, deleted, or
+ * unreadable. Ids are de-duplicated, and the lookup runs in batches to keep
+ * individual Couch requests small.
+ *
+ * This yields per batch rather than returning everything at once, so a caller
+ * working through hundreds of thousands of ids can report progress instead of
+ * going quiet for the whole lookup.
+ */
+export async function* streamDeviceBatchesByIds(
+  connections: DbConnections,
+  deviceIds: string[]
+): AsyncIterableIterator<DeviceBatch> {
+  const db = connections.couch.use(couchDevicesSetup.name)
+  const unique = [...new Set(deviceIds)]
+
+  for (let i = 0; i < unique.length; i += FETCH_BATCH_SIZE) {
+    const keys = unique.slice(i, i + FETCH_BATCH_SIZE)
+    const response = await db.fetch({ keys })
+
+    const deviceRows: DeviceRow[] = []
+    for (const row of response.rows) {
+      // Missing and deleted ids come back as rows without a document:
+      if ('error' in row || row.doc == null) continue
+      const couchDevice = asMaybe(asCouchDevice)(row.doc)
+      if (couchDevice == null) continue
+      deviceRows.push(makeDeviceRow(db, couchDevice))
+    }
+    yield { deviceRows, idsRead: Math.min(i + keys.length, unique.length) }
+  }
+}
+
 /**
  * Finds all the devices that have logged into this account.
  */
@@ -343,3 +422,87 @@ export async function* streamDevicesByLocation(
     yield makeDeviceRow(db, couchDevice)
   }
 }
+
+/**
+ * Builds a CouchDB start key of [apiKey, country, region, city], stopping at the
+ * first missing location field so the range stays a valid prefix.
+ */
+function makeApiKeyLocationStartKey(
+  apiKey: string,
+  location: { country?: string; region?: string; city?: string }
+): string[] {
+  const key = [apiKey]
+  for (const part of [location.country, location.region, location.city]) {
+    if (part == null) break
+    key.push(part)
+  }
+  return key
+}
+
+/**
+ * What the `apiKeyLocation` view stores alongside each device, so that an
+ * audience query can work from the index alone.
+ */
+export interface DeviceSummary {
+  deviceId: string
+  apiKey: string
+  region: string
+  city: string
+  deviceToken: string | undefined
+  ignoreMarketing: boolean
+  visited: Date
+}
+
+const asViewValue = asObject({
+  region: asOptional(asString, ''),
+  city: asOptional(asString, ''),
+  deviceToken: asOptional(asString),
+  ignoreMarketing: asOptional(asBoolean, false),
+  visited: asOptional(asDate, () => new Date(0))
+})
+
+// Rows to pull per request while paging through the view:
+const VIEW_PAGE_SIZE = 2048
+
+/**
+ * Streams a summary of every device registered under an API key, optionally
+ * narrowed by location.
+ *
+ * This reads the view rows only. Fetching the documents would mean pulling
+ * hundreds of megabytes to read a handful of fields, since the view spans a
+ * whole country and the location filters are applied afterwards.
+ */
+export async function* streamDeviceSummariesByApiKeyLocation(
+  connections: DbConnections,
+  apiKey: string,
+  location: { country?: string; region?: string; city?: string }
+): AsyncIterableIterator<DeviceSummary> {
+  const db = connections.couch.use(couchDevicesSetup.name)
+  const startKey = makeApiKeyLocationStartKey(apiKey, location)
+  const endKey = [...startKey, 'zzzzzz']
+
+  let params: object = { start_key: startKey }
+  while (true) {
+    const response = await db.view('apiKeyLocation', 'apiKeyLocation', {
+      reduce: false,
+      include_docs: false,
+      limit: VIEW_PAGE_SIZE,
+      end_key: endKey,
+      ...params
+    })
+    const { rows } = response
+    if (rows.length === 0) return
+
+    for (const row of rows) {
+      const value = asMaybe(asViewValue)(row.value)
+      if (value == null) continue
+      yield { ...value, deviceId: row.id, apiKey }
+    }
+
+    if (rows.length < VIEW_PAGE_SIZE) return
+    // Resume after the last row, which needs the doc id to break ties between
+    // devices sharing a location:
+    const last = rows[rows.length - 1]
+    params = { start_key: last.key, start_key_doc_id: last.id, skip: 1 }
+  }
+}

src/server/index.ts

@@ -5,6 +5,7 @@ import { makeExpressRoute } from 'serverlet/express'
 import { setupDatabases } from '../db/couchSetup'
 import { makeConnections, serverConfig } from '../serverConfig'
 import { logger } from '../util/logger'
+import { makeMarketingToolRouter } from './marketing/marketingTool'
 import { withLogging } from './middleware/withLogging'
 import { allRoutes } from './urls'
 
@@ -23,6 +24,12 @@ async function main(): Promise<void> {
   // Set up Express:
   const app = express()
   app.enable('trust proxy')
+
+  // The marketing tool API, mounted before the Serverlet catch-all. It comes
+  // ahead of the body parser below because it installs its own with a larger
+  // limit, and whichever parser runs first wins:
+  app.use('/marketing', makeMarketingToolRouter(connections))
+
   app.use(express.json({ limit: '1mb' }))
   app.use('/', makeExpressRoute(server))
 

src/server/marketing/locationFilter.ts

@@ -0,0 +1,99 @@
+import { Device } from '../../types/pushTypes'
+
+/**
+ * An include/exclude filter over the city and region a device is located in.
+ * The lists hold normalized values (see `parseLocationList`).
+ */
+export interface LocationFilter {
+  cityInclude: string[]
+  cityExclude: string[]
+  regionInclude: string[]
+  regionExclude: string[]
+}
+
+/**
+ * Why a device cannot receive a marketing push.
+ */
+export type DeviceSkipReason =
+  | 'ignore-marketing'
+  | 'unregistered'
+  | 'invalid-token'
+
+// Firebase tokens are alphanumerics plus these separators. Anything else is a
+// corrupt row that would only fail at delivery time:
+const VALID_TOKEN = /^[a-zA-Z0-9_\-:]+$/
+
+/**
+ * Normalizes one include/exclude box: each line is a separate value, blank
+ * lines are dropped, and matching is case-insensitive.
+ */
+export function parseLocationList(lines: string[]): string[] {
+  const out = new Set<string>()
+  for (const line of lines) {
+    const value = line.trim().toLowerCase()
+    if (value !== '') out.add(value)
+  }
+  return [...out]
+}
+
+/**
+ * Decides whether a device's location passes the filter. Values within one box
+ * are OR'd together, and the boxes are AND'ed with each other. An empty include
+ * box means "no restriction", while an empty exclude box excludes nothing.
+ *
+ * Country is deliberately absent: the caller selects it through the Couch view
+ * prefix, so re-checking it here could only disagree with the query.
+ */
+export function matchesLocationFilter(
+  location: Device['location'],
+  filter: LocationFilter
+): boolean {
+  if (location == null) return false
+  const city = location.city.trim().toLowerCase()
+  const region = location.region.trim().toLowerCase()
+
+  return (
+    passes(city, filter.cityInclude, filter.cityExclude) &&
+    passes(region, filter.regionInclude, filter.regionExclude)
+  )
+}
+
+function passes(value: string, include: string[], exclude: string[]): boolean {
+  if (include.length > 0 && !include.includes(value)) return false
+  return !exclude.includes(value)
+}
+
+/** The parts of a device that decide whether it can be sent to. */
+export type SendableFields = Pick<
+  Device,
+  'apiKey' | 'deviceToken' | 'ignoreMarketing'
+>
+
+/**
+ * Reports why a device cannot be sent to, or undefined if it can.
+ *
+ * The `targetApiKeys` are the app keys the marketing key may send to. Location
+ * queries get their scoping from the Couch view, but a send driven by a
+ * caller-supplied list of device ids does not, and the publish daemon resolves
+ * Firebase credentials from whatever key the device itself carries.
+ *
+ * This takes only the fields it reads so that queries can apply it to view
+ * rows, and sends to whole documents, without the two drifting apart.
+ */
+export function getDeviceSkipReason(
+  device: SendableFields,
+  targetApiKeys: ReadonlySet<string>
+): DeviceSkipReason | undefined {
+  const { apiKey, deviceToken, ignoreMarketing } = device
+
+  if (ignoreMarketing) return 'ignore-marketing'
+  if (
+    apiKey == null ||
+    !targetApiKeys.has(apiKey) ||
+    deviceToken == null ||
+    deviceToken.trim() === ''
+  ) {
+    return 'unregistered'
+  }
+  if (!VALID_TOKEN.test(deviceToken)) return 'invalid-token'
+}