[pull] series/0.19 from disneystreaming:series/0.19

CHANGELOG.md

@@ -13,6 +13,8 @@ Thank you!
 - codegen: Fix `BigInteger`/`BigDecimal` default values and `@range` bounds losing precision when they don't fit in an `Int`/`Double` (e.g. `4294967296` becoming `0`, `9007199254740993` becoming `9007199254740992`), in [#2002](https://github.com/disneystreaming/smithy4s/pull/2002), fixing [#2001](https://github.com/disneystreaming/smithy4s/issues/2001). Values are now carried exactly and rendered as `scala.math.BigDecimal("...")`/`scala.math.BigInt("...")` string literals. Integer `Document` defaults beyond the `Long` range are rendered with `Document.fromBigDecimal` instead of wrapping. As a side effect, `@range` bounds on integer literals now print without a trailing `.0` in validation error messages (e.g. `Input must be >= 1` instead of `>= 1.0`).
 - Fix `@range` validation losing precision by converting every value through `Double` before comparing it to the bounds, in [#2004](https://github.com/disneystreaming/smithy4s/pull/2004), fixing [#2003](https://github.com/disneystreaming/smithy4s/issues/2003). `Long` values above 2^53, `BigInt` values beyond the double range (which used to be rejected as infinity) and `BigDecimal` values are now compared exactly. The matching codegen fix for `@range` bounds rendered through `Double` is in [#2002](https://github.com/disneystreaming/smithy4s/pull/2002). Validation messages for integral inputs no longer render a trailing `.0` (e.g. `but was 11` instead of `but was 11.0`).
 - Fix `@http` routes never matching when a greedy label (`{foo+}`) is followed by a static segment (e.g. `/base/{foo+}/end`) in [#1998](https://github.com/disneystreaming/smithy4s/pull/1998). `matchPath` only accumulated greedy segments when the greedy label was the last path segment, so such routes always 404'd.
+- aws: Resolve the SigV4 signing name and the endpoint host independently in [#1596](https://github.com/disneystreaming/smithy4s/pull/1596). The signing name now comes from `aws.auth#sigv4`'s `name`, while the host keeps using `endpointPrefix`, falling back to `arnNamespace` when it is absent. Both previously used `endpointPrefix`, so services where the two differ were rejected with "Credential should be scoped to correct service" ([#1568](https://github.com/disneystreaming/smithy4s/issues/1568)), and services without an `endpointPrefix` derived the host from the *operation* name, e.g. `ListRegions.us-east-1.amazonaws.com` ([#1532](https://github.com/disneystreaming/smithy4s/issues/1532)).
+- codegen: Skip traits from the `aws.endpoints` namespace when generating hints. They have no published Scala bindings, so services carrying them (e.g. the AWS Account API, via `aws.endpoints#standardPartitionalEndpoints`) generated code that did not compile. This mirrors the existing treatment of `smithy.rules`.
 
 # 0.19.11
 

modules/aws-http4s/src/smithy4s/aws/AwsClient.scala

@@ -66,22 +66,47 @@ object AwsClient {
       val service: smithy4s.Service[Alg]
   ) {
 
+    /**
+      * The host prefix of the standard-partition endpoint, which is not
+      * necessarily the name used when signing (see `AwsSigning.signingName`):
+      * SES v2 is hosted at `email.<region>.amazonaws.com` but signs as `ses`.
+      *
+      * `endpointPrefix` is the field meant for this: it "identifies which
+      * endpoint in a given region should be used to connect to the service",
+      * resolving as `{endpointPrefix}.{region}.{dnsSuffix}`. See
+      * https://smithy.io/2.0/aws/aws-core.html#endpointprefix
+      *
+      * It is optional, though, and some services (the Account API, for one) omit
+      * it. We then fall back to `arnNamespace`, which is constrained to
+      * `^[a-z0-9.\-]{1,63}$` and defaults to the lowercased service shape name,
+      * making it a DNS-safe stand-in. See
+      * https://smithy.io/2.0/aws/aws-core.html#arnnamespace
+      *
+      * This only covers the `{prefix}.{region}.amazonaws.com` pattern; FIPS,
+      * dual-stack, non-standard partitions (`amazonaws.com.cn`, GovCloud) and
+      * operation-level endpoints all require evaluating
+      * `smithy.rules#endpointRuleSet`, which we do not do.
+      */
+    private val hostPrefix: String =
+      awsService.endpointPrefix
+        .orElse(awsService.arnNamespace.map(_.value))
+        .getOrElse(service.id.name)
+        .toLowerCase()
+
     private def compiler[F[_]: Async: Compression: Hashing](
         awsEnv: AwsEnvironment[F]
     ): service.FunctorEndpointCompiler[F] = {
 
-      def baseRequest(endpoint: OperationSchema[_, _, _, _, _]): F[HttpRequest[Blob]] = {
+      def baseRequest(@annotation.unused endpoint: OperationSchema[_, _, _, _, _]): F[HttpRequest[Blob]] = {
         awsEnv.region.map { region =>
-          val endpointPrefix = awsService.endpointPrefix.getOrElse(endpoint.id.name)
           val baseUri = HttpUri.absolute(
             scheme = HttpUriScheme.Https,
-            host = s"$endpointPrefix.$region.amazonaws.com",
+            host = s"$hostPrefix.$region.amazonaws.com",
             port = None,
             path = IndexedSeq.empty,
             queryParams = IndexedSeq.empty,
             pathParams = None
           )
-          // Uri.unsafeFromString(s"https://$endpointPrefix.$region.amazonaws.com/")
           HttpRequest(HttpMethod.POST, baseUri, Map.empty, Blob.empty)
         }
       }

modules/aws-http4s/src/smithy4s/aws/internals/AwsSigning.scala

@@ -67,16 +67,10 @@ private[aws] object AwsSigning {
       endpointHints: Hints,
       awsEnvironment: AwsEnvironment[F]
   ): Client[F] => Client[F] = {
-    val endpointPrefix = serviceHints
-      .get(_root_.aws.api.Service)
-      .flatMap(_.endpointPrefix)
-      .getOrElse(serviceId.name)
-      .toLowerCase()
-
     val sign = signingFunction(
       serviceId.name,
       endpointId.name,
-      endpointPrefix,
+      signingName(serviceId, serviceHints),
       awsEnvironment.timestamp,
       awsEnvironment.credentials,
       awsEnvironment.region
@@ -89,10 +83,44 @@ private[aws] object AwsSigning {
       }
   }
 
+  /**
+    * The name used in the credential scope when signing requests, which is not
+    * necessarily the same as the host prefix (see `AwsClient`): SES v2, for
+    * instance, is hosted at `email.<region>.amazonaws.com` but signs as `ses`.
+    *
+    * `aws.auth#sigv4`'s `name` is, per its own definition, "the signature
+    * version 4 service signing name to use in the credential scope when signing
+    * requests", so it wins. It also states that the value SHOULD match
+    * `arnNamespace`, which is why that is preferred over `endpointPrefix` --
+    * the latter is documented as being unstable and non-unique, and MUST NOT be
+    * used for anything other than resolving endpoints. See
+    * https://smithy.io/2.0/aws/aws-auth.html#aws-auth-sigv4-trait and
+    * https://smithy.io/2.0/aws/aws-core.html#endpointprefix
+    */
+  private[internals] def signingName(
+      serviceId: ShapeId,
+      serviceHints: Hints
+  ): String = {
+    val fromSigv4 = serviceHints.get(_root_.aws.auth.Sigv4).map(_.name)
+
+    val fromAwsService = serviceHints
+      .get(_root_.aws.api.Service)
+      .flatMap { awsService =>
+        awsService.arnNamespace
+          .map(_.value)
+          .orElse(awsService.endpointPrefix)
+      }
+
+    fromSigv4
+      .orElse(fromAwsService)
+      .getOrElse(serviceId.name)
+      .toLowerCase()
+  }
+
   private[internals] def signingFunction[F[_]: Concurrent](
       serviceName: String,
       operationName: String,
-      endpointPrefix: String,
+      signingName: String,
       timestamp: F[Timestamp],
       credentials: F[AwsCredentials],
       region: F[AwsRegion]
@@ -153,7 +181,7 @@ private[aws] object AwsSigning {
         }
         .flatMap { case ((payloadHash, preparedRequest), pathString) =>
           val awsHeadersF = (timestamp, credentials, region).mapN { case (timestamp, credentials, region) =>
-            val credentialsScope = s"${timestamp.conciseDate}/$region/$endpointPrefix/aws4_request"
+            val credentialsScope = s"${timestamp.conciseDate}/$region/$signingName/aws4_request"
             val queryParams: Vector[(String, String)] =
               request.uri.query.toVector.sorted.map { case (k, v) => k -> v.getOrElse("") }
             val canonicalQueryString =
@@ -202,7 +230,7 @@ private[aws] object AwsSigning {
               credentials.secretAccessKey,
               timestamp.conciseDate,
               region.value,
-              endpointPrefix
+              signingName
             )
             val stringToSign = List[String](
               algorithm,

modules/aws-http4s/test/src/smithy4s/aws/AwsEndpointAndSigningNameTest.scala

@@ -0,0 +1,153 @@
+/*
+ *  Copyright 2021-2026 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package smithy4s.aws
+
+import cats.effect.IO
+import cats.effect.Resource
+import cats.syntax.all._
+import com.amazonaws.dynamodb.DynamoDB
+import org.http4s.Request
+import org.http4s.Response
+import org.http4s.Status
+import org.http4s.client.Client
+import smithy4s.aws.kernel.AwsCredentials
+import smithy4s.aws.kernel.Timestamp
+import weaver._
+
+/**
+  * The host and the sigv4 signing name are resolved from *different* fields and
+  * are not interchangeable: SES v2 is reached at `email.<region>.amazonaws.com`
+  * but must be signed as `ses`, or AWS rejects the request with "Credential
+  * should be scoped to correct service".
+  *
+  * These tests pin both halves for each combination of `endpointPrefix`,
+  * `arnNamespace` and `aws.auth#sigv4` we have seen in the wild.
+  */
+object AwsEndpointAndSigningNameTest extends SimpleIOSuite with Compat {
+
+  private val credentials = AwsCredentials.Default(
+    accessKeyId = "AKIAIOSFODNN7EXAMPLE",
+    secretAccessKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
+    sessionToken = None
+  )
+
+  /** 2024-09-27T00:00:00Z -- fixed so the credential scope is deterministic. */
+  private val timestamp = Timestamp(1727395200L, 0)
+
+  /**
+    * Runs one operation against a stub client and returns the request that
+    * would have gone out to AWS.
+    */
+  private def capture(
+      run: AwsEnvironment[IO] => Resource[IO, Any]
+  ): IO[Request[IO]] =
+    IO.ref(Option.empty[Request[IO]]).flatMap { ref =>
+      val httpClient = Client[IO] { req =>
+        Resource.eval(ref.set(Some(req))).as(Response[IO](Status.Forbidden))
+      }
+
+      val awsEnv = AwsEnvironment.make[IO](
+        httpClient,
+        IO.pure(AwsRegion.US_EAST_1),
+        IO.pure(credentials),
+        IO.pure(timestamp)
+      )
+
+      run(awsEnv).use_.attempt *>
+        ref.get.flatMap(
+          _.liftTo[IO](new AssertionError("no request was sent"))
+        )
+    }
+
+  private def host(request: Request[IO]): Option[String] =
+    request.uri.host.map(_.renderString)
+
+  /** The `<service>` part of `Credential=<key>/<date>/<region>/<service>/aws4_request`. */
+  private def signingName(request: Request[IO]): Option[String] =
+    request.headers
+      .get(org.typelevel.ci.CIString("Authorization"))
+      .map(_.head.value)
+      .flatMap { auth =>
+        "Credential=[^/]+/[^/]+/[^/]+/([^/]+)/aws4_request".r
+          .findFirstMatchIn(auth)
+          .map(_.group(1))
+      }
+
+  test(
+    "endpointPrefix and sigv4 name differ: host uses the prefix, signing uses sigv4 (SES v2)"
+  ) {
+    capture { awsEnv =>
+      AwsClient(smithy4s.example.aws.PrefixDiffersFromSigningName, awsEnv)
+        .evalMap(_.doThing())
+    }.map { request =>
+      expect.same(host(request), Some("email.us-east-1.amazonaws.com")) &&
+      expect.same(signingName(request), Some("ses"))
+    }
+  }
+
+  test(
+    "dotted endpointPrefix: host keeps the dots, signing uses sigv4 (SageMaker, #1568)"
+  ) {
+    capture { awsEnv =>
+      AwsClient(smithy4s.example.aws.DottedPrefix, awsEnv)
+        .evalMap(_.doThing())
+    }.map { request =>
+      expect.same(
+        host(request),
+        Some("api.sagemaker.us-east-1.amazonaws.com")
+      ) &&
+      expect.same(signingName(request), Some("sagemaker"))
+    }
+  }
+
+  test(
+    "no endpointPrefix: both fall back to arnNamespace, never the operation name (Account, #1532)"
+  ) {
+    capture { awsEnv =>
+      AwsClient(smithy4s.example.aws.NoEndpointPrefix, awsEnv)
+        .evalMap(_.doThing())
+    }.map { request =>
+      expect.same(host(request), Some("account.us-east-1.amazonaws.com")) &&
+      expect.same(signingName(request), Some("account"))
+    }
+  }
+
+  test(
+    "no sigv4 trait: signing falls back to arnNamespace rather than endpointPrefix"
+  ) {
+    capture { awsEnv =>
+      AwsClient(smithy4s.example.aws.NoSigv4, awsEnv)
+        .evalMap(_.doThing())
+    }.map { request =>
+      expect.same(
+        host(request),
+        Some("endpointprefix.us-east-1.amazonaws.com")
+      ) &&
+      expect.same(signingName(request), Some("arnnamespace"))
+    }
+  }
+
+  test("all three names agree: nothing changes (DynamoDB)") {
+    capture { awsEnv =>
+      AwsClient(DynamoDB, awsEnv).evalMap(_.listTables())
+    }.map { request =>
+      expect.same(host(request), Some("dynamodb.us-east-1.amazonaws.com")) &&
+      expect.same(signingName(request), Some("dynamodb"))
+    }
+  }
+
+}

modules/bootstrapped/src/generated/smithy4s/example/aws/DoThingInput.scala

@@ -0,0 +1,20 @@
+package smithy4s.example.aws
+
+import smithy4s.Hints
+import smithy4s.Schema
+import smithy4s.ShapeId
+import smithy4s.ShapeTag
+import smithy4s.schema.Schema.constant
+
+final case class DoThingInput()
+
+object DoThingInput extends ShapeTag.Companion[DoThingInput] {
+  val id: ShapeId = ShapeId("smithy4s.example.aws", "DoThingInput")
+
+  val hints: Hints = Hints(
+    Hints.dynamic(ShapeId("smithy.api", "input"), smithy4s.Document.obj()),
+  )
+
+
+  implicit val schema: Schema[DoThingInput] = constant(DoThingInput()).withId(id).addHints(hints)
+}

modules/bootstrapped/src/generated/smithy4s/example/aws/DoThingOutput.scala

@@ -0,0 +1,20 @@
+package smithy4s.example.aws
+
+import smithy4s.Hints
+import smithy4s.Schema
+import smithy4s.ShapeId
+import smithy4s.ShapeTag
+import smithy4s.schema.Schema.constant
+
+final case class DoThingOutput()
+
+object DoThingOutput extends ShapeTag.Companion[DoThingOutput] {
+  val id: ShapeId = ShapeId("smithy4s.example.aws", "DoThingOutput")
+
+  val hints: Hints = Hints(
+    Hints.dynamic(ShapeId("smithy.api", "output"), smithy4s.Document.obj()),
+  )
+
+
+  implicit val schema: Schema[DoThingOutput] = constant(DoThingOutput()).withId(id).addHints(hints)
+}