From 65b60e3fe898e6e8fc487e2ffe31232ac706f174 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:59:13 +0200 Subject: [PATCH 01/25] Backend: read-only columns, @Lob, CrudRepository bulk methods, produces/consumes, @CrossOrigin Part of #6153: @Column(insertable/updatable), @Lob accepted on string and byte fields, CrudRepository delete/deleteAll/saveAll/findAllById/flush and default methods in repository interfaces, consumes/produces on the mapping annotations (415/406), a per-route @CrossOrigin policy that replaces the global one for its route, and ResponseStatusException(int). Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/annotations/Column.java | 12 ++ .../src/com/codename1/annotations/db/Lob.java | 41 +++++ .../src/com/codename1/impl/orm/Attribute.java | 10 ++ .../com/codename1/impl/orm/SessionImpl.java | 13 +- .../orm/repository/CrudRepository.java | 23 +++ .../codename1/backend/RoutePolicyTest.java | 89 +++++++++++ .../processors/OrmAnnotationProcessor.java | 16 +- .../maven/processors/RepositoryGenerator.java | 16 +- .../RestControllerAnnotationProcessor.java | 128 ++++++++++++++- .../OrmAnnotationProcessorTest.java | 56 ++++++- ...RestControllerAnnotationProcessorTest.java | 75 ++++++++- .../src/com/codename1/backend/Backend.java | 21 +++ .../src/com/codename1/backend/Cors.java | 60 ++++++- .../backend/ResponseStatusException.java | 10 +- .../backend/annotations/CrossOrigin.java | 64 ++++++++ .../backend/annotations/DeleteMapping.java | 7 + .../backend/annotations/GetMapping.java | 7 + .../backend/annotations/PatchMapping.java | 7 + .../backend/annotations/PostMapping.java | 7 + .../backend/annotations/PutMapping.java | 7 + .../backend/annotations/RequestMapping.java | 9 ++ .../com/codename1/backend/web/CorsRoutes.java | 38 +++++ .../com/codename1/backend/web/MediaTypes.java | 146 ++++++++++++++++++ 23 files changed, 844 insertions(+), 18 deletions(-) create mode 100644 CodenameOne/src/com/codename1/annotations/db/Lob.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java create mode 100644 vm/backend/src/com/codename1/backend/annotations/CrossOrigin.java create mode 100644 vm/backend/src/com/codename1/backend/web/CorsRoutes.java create mode 100644 vm/backend/src/com/codename1/backend/web/MediaTypes.java diff --git a/CodenameOne/src/com/codename1/annotations/Column.java b/CodenameOne/src/com/codename1/annotations/Column.java index 2d7ec073a41..7a1bdc0a130 100644 --- a/CodenameOne/src/com/codename1/annotations/Column.java +++ b/CodenameOne/src/com/codename1/annotations/Column.java @@ -58,4 +58,16 @@ /// engine spells differently (`TEXT`/`VARCHAR`, `REAL`/`DOUBLE PRECISION`, /// `BLOB`/`BYTEA`). String type() default ""; + /// When false the column is left out of every `INSERT` the ORM writes, so the + /// row starts with the column's database default. The field's value at + /// insert time is ignored, not written. + boolean insertable() default true; + /// When false the column is left out of every `UPDATE` the ORM writes for a + /// changed entity, so a save never overwrites it. The column can still be + /// written by a query of the application's own -- a `@Modifying @Query` that + /// claims a row with a compare-and-swap, for example -- which is what makes + /// `insertable = false, updatable = false` a column only that statement owns. + /// A change made to the field alone is ignored: it neither writes the column + /// nor marks the entity dirty. + boolean updatable() default true; } diff --git a/CodenameOne/src/com/codename1/annotations/db/Lob.java b/CodenameOne/src/com/codename1/annotations/db/Lob.java new file mode 100644 index 00000000000..d673cc369b8 --- /dev/null +++ b/CodenameOne/src/com/codename1/annotations/db/Lob.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.annotations.db; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Marks a `String`, `char[]` or `byte[]` field as a large object, as JPA's +/// `@Lob` does. +/// +/// It changes nothing about the column, because every string and byte column +/// is already one: MySQL gets `LONGTEXT` and `LONGBLOB`, and SQLite and +/// PostgreSQL leave `TEXT` and `BLOB`/`BYTEA` unbounded. It exists so an entity +/// ported from JPA keeps the annotation, and the build rejects it on a field of +/// any other type, as JPA would. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.FIELD) +@com.codename1.impl.SharedWithBackend +public @interface Lob { +} diff --git a/CodenameOne/src/com/codename1/impl/orm/Attribute.java b/CodenameOne/src/com/codename1/impl/orm/Attribute.java index 2decf4c0af0..f42db495bf7 100644 --- a/CodenameOne/src/com/codename1/impl/orm/Attribute.java +++ b/CodenameOne/src/com/codename1/impl/orm/Attribute.java @@ -51,6 +51,10 @@ public final class Attribute { /// itself is TEXT storage through TemporalConverter; this picks the DDL type /// and what an existing column may be. public final int temporal; + /// False for `@Column(insertable = false)`: no INSERT names the column. + public final boolean insertable; + /// False for `@Column(updatable = false)`: no dirty-checking UPDATE names it. + public final boolean updatable; /// The kind that decides SQL semantics -- collation, LIKE, the text /// functions -- as opposed to [#kind], which decides how a value is bound and /// read. A `@Temporal` column exchanges text through its converter but is a @@ -68,6 +72,12 @@ public Attribute(String field, String column, int kind, boolean id, boolean gene } public Attribute(String field, String column, int kind, boolean id, boolean generated, boolean nullable, boolean version, String declaredType, int temporal) { + this(field, column, kind, id, generated, nullable, version, declaredType, temporal, true, true); + } + public Attribute(String field, String column, int kind, boolean id, boolean generated, boolean nullable, + boolean version, String declaredType, int temporal, boolean insertable, boolean updatable) { + this.insertable = insertable; + this.updatable = updatable; this.temporal = temporal; this.declaredType = declaredType; this.field = field; diff --git a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java index d9864081a77..ca72c365f3f 100644 --- a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java +++ b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java @@ -2899,7 +2899,7 @@ private void insert(Entry entry, List inserted) { model.set(entry.entity, version, Long.valueOf(0)); } for (int i = 0; i < attrs.length; i++) { - if (attrs[i].generated) { + if (!inserted(attrs[i])) { continue; } if (!args.isEmpty()) { @@ -2948,7 +2948,7 @@ private void insert(Entry entry, List inserted) { entry.snapshot = snapshot(model, entry.entity); int arg = 0; for (int i = 0; i < attrs.length; i++) { - if (!attrs[i].generated) { + if (inserted(attrs[i])) { Object value = args.get(arg++); entry.snapshot[i] = value instanceof byte[] ? ((byte[]) value).clone() : value; } @@ -2956,6 +2956,11 @@ private void insert(Entry entry, List inserted) { model.lifecycle(entry.entity, 1); cascadePersist(entry); } + /// Whether an INSERT names the column: not a generated key, and not a + /// `@Column(insertable = false)` one, whose row starts at the database default. + private static boolean inserted(Attribute a) { + return !a.generated && a.insertable; + } private void completeInsert(Entry entry) { checkManagedIdentity(entry); checkTransientAssociations(entry); @@ -3067,7 +3072,7 @@ private boolean update(Entry entry, List inserted, boolean checkCollectio } boolean dirty = false; for (int i = 0; i < attrs.length; i++) { - if (!attrs[i].id && !attrs[i].version && !same(now[i], entry.snapshot[i])) { + if (!attrs[i].id && !attrs[i].version && attrs[i].updatable && !same(now[i], entry.snapshot[i])) { dirty = true; } } @@ -3098,7 +3103,7 @@ private boolean update(Entry entry, List inserted, boolean checkCollectio List args = new ArrayList(); StringBuilder set = new StringBuilder(); for (int i = 0; i < attrs.length; i++) { - if (attrs[i].id || attrs[i].version || same(now[i], entry.snapshot[i])) { + if (attrs[i].id || attrs[i].version || !attrs[i].updatable || same(now[i], entry.snapshot[i])) { continue; } if (!args.isEmpty()) { diff --git a/CodenameOne/src/com/codename1/orm/repository/CrudRepository.java b/CodenameOne/src/com/codename1/orm/repository/CrudRepository.java index 6a1f97d43c4..c8922401b54 100644 --- a/CodenameOne/src/com/codename1/orm/repository/CrudRepository.java +++ b/CodenameOne/src/com/codename1/orm/repository/CrudRepository.java @@ -33,4 +33,27 @@ public interface CrudRepository { java.util.List findAll(); long count(); boolean existsById(K id); + + /// Deletes one entity, managed or detached, as Spring Data's `delete` does: a + /// detached instance is merged first, so the row it stands for is removed. An + /// entity whose row no longer exists is ignored. Requires a transaction. + void delete(E entity); + + /// Deletes every entity of this type, one by one, so cascades and lifecycle + /// callbacks run as they do for `delete`. Requires a transaction. + void deleteAll(); + + /// Deletes each of the given entities, as `delete` does. Requires a transaction. + void deleteAll(Iterable entities); + + /// Saves each of the given entities, as `save` does, and answers the managed + /// results in the same order. + java.util.List saveAll(Iterable entities); + + /// The entities whose ids are given, in the order given; an id with no row is + /// skipped, so the list can be shorter than the input. + java.util.List findAllById(Iterable ids); + + /// Writes pending changes of the caller's session now, rather than at commit. + void flush(); } diff --git a/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java b/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java new file mode 100644 index 00000000000..9c502185fb4 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import com.codename1.backend.web.MediaTypes; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// The policy a `@CrossOrigin` route builds, and the media-type matching behind +/// `consumes`/`produces`. +class RoutePolicyTest { + private static HttpServer.Request request(String method, String... headers) throws Exception { + StringBuilder text = new StringBuilder(method + " /x HTTP/1.1\r\n"); + int[] slices = new int[headers.length * 4]; + for (int i = 0; i < headers.length; i++) { + int colon = headers[i].indexOf(':'); + String name = headers[i].substring(0, colon); + String value = headers[i].substring(colon + 1).trim(); + slices[i * 4] = text.length(); + slices[i * 4 + 1] = name.length(); + text.append(name).append(": "); + slices[i * 4 + 2] = text.length(); + slices[i * 4 + 3] = value.length(); + text.append(value).append("\r\n"); + } + byte[] raw = text.append("\r\n").toString().getBytes("UTF-8"); + return new HttpServer.Request(method, "/x", "HTTP/1.1", raw, slices, headers.length, null); + } + + @Test + void aRoutePolicyAdmitsOnlyItsOrigins() throws Exception { + Cors policy = Cors.policy(new String[] {"https://app.example.com"}, new String[0], + new String[] {"*"}, new String[] {"X-Total"}, true, 600); + assertNull(policy.reject(request("GET", "Origin: https://app.example.com", "Host: api.example.com"))); + HttpServer.Response denied = policy.reject(request("GET", "Origin: https://evil.example", "Host: api.example.com")); + assertNotNull(denied); + assertEquals(403, denied.getStatus()); + HttpServer.Response preflight = policy.preflight(request("OPTIONS", "Origin: https://app.example.com", + "Host: api.example.com", "Access-Control-Request-Method: DELETE")); + assertEquals(204, preflight.getStatus()); + assertTrue(preflight.hasHeader("Access-Control-Allow-Origin")); + } + + @Test + void credentialsWithAnyOriginIsRefused() { + assertThrows(IllegalArgumentException.class, () -> Cors.policy(new String[] {"*"}, new String[0], + new String[] {"*"}, new String[0], true, 600)); + assertThrows(IllegalArgumentException.class, () -> Cors.policy(new String[] {"https://a.example/path"}, + new String[0], new String[] {"*"}, new String[0], false, 600)); + } + + @Test + void mediaTypesMatchWithWildcardsAndParameters() { + assertTrue(MediaTypes.consumable("application/json; charset=UTF-8", "application/json")); + assertTrue(MediaTypes.consumable("text/csv", "application/json,text/*")); + assertFalse(MediaTypes.consumable(null, "application/json")); + assertFalse(MediaTypes.consumable("application/jsonx", "application/json")); + assertTrue(MediaTypes.acceptable(null, "application/json")); + assertTrue(MediaTypes.acceptable("*/*", "application/json")); + assertFalse(MediaTypes.acceptable("application/json;q=0", "application/json")); + assertFalse(MediaTypes.acceptable("image/png", "application/json")); + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java index 056bf99f25f..737f5f8036b 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java @@ -481,6 +481,12 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from if(converterClass==null || !hasPublicNoArgConstructor(converterClass)) ctx.error(cls,"Converter requires a public no-arg constructor: "+pf.converter); if(f.getAnnotation(ID_DESC)!=null || pf.version || ec.embeddedId!=null && pf.fieldName.startsWith(ec.embeddedId+".")) ctx.error(cls,"Identifier and version fields cannot declare converters"); } + // @Lob is accepted for a ported JPA entity and changes no column: every + // string and byte column is already a large one (see Lob). Only the type is checked. + if(f.getAnnotation("Lcom/codename1/annotations/db/Lob;")!=null && !("java.lang.String".equals(pf.javaType) + || "byte[]".equals(pf.javaType) || "char[]".equals(pf.javaType) || pf.converter!=null)) { + ctx.error(cls,"@Lob applies to a String, char[] or byte[] field: "+pf.fieldName); + } AnnotationValues temporal=f.getAnnotation("Lcom/codename1/annotations/db/Temporal;"); if(temporal!=null) { // A native date or timestamp column, opted into per field. It is @@ -560,6 +566,11 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from colName = col.getString("name"); colType = col.getString("type"); nullable = col.getBoolOrDefault("nullable", true); + pf.insertable = col.getBoolOrDefault("insertable", true); + pf.updatable = col.getBoolOrDefault("updatable", true); + if ((!pf.insertable || !pf.updatable) && (f.getAnnotation(ID_DESC) != null || pf.version)) { + ctx.error(cls, "@Column(insertable/updatable=false) cannot be used on an @Id or @Version field: " + pf.fieldName); + } } pf.columnName = (colName == null || colName.length() == 0) ? physical(pf.fieldName) : physical(fieldPath.prefix)+colName; @@ -1535,7 +1546,8 @@ private static String generateModelSource(EntityClass ec, boolean backend) { .append(", ").append(f.isId).append(", ").append(f.isId && f.autoIncrement) .append(", ").append(f.nullable).append(", ").append(f.version).append(", ") .append(f.explicitSqlType==null?"null":"\""+escape(f.explicitSqlType)+"\"") - .append(f.temporal==0?"":", "+f.temporal).append(")") + .append(f.temporal==0 && f.insertable && f.updatable?"":", "+f.temporal) + .append(f.insertable && f.updatable?"":", "+f.insertable+", "+f.updatable).append(")") .append(i + 1 < ec.fields.size() ? ",\n" : "\n"); } sb.append(" };\n public Class<").append(ec.binaryName).append("> type() { return ") @@ -3165,6 +3177,8 @@ static final class PersistedField { boolean autoIncrement; boolean version; boolean unique; + /// @Column(insertable/updatable); both true unless the developer said otherwise. + boolean insertable = true, updatable = true; RelationField relation; int relationPart; PropertyTypeKind kind; diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/RepositoryGenerator.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/RepositoryGenerator.java index afe10514f4c..57e77579e15 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/RepositoryGenerator.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/RepositoryGenerator.java @@ -74,7 +74,9 @@ static String generate(AnnotatedClass repository, Map custom = new HashSet(); for (MethodInfo method : repository.getMethods()) { if (method.isStatic() || method.isSynthetic()) continue; - if (!method.isAbstract()) throw invalid(method.getName() + ": default methods are unsupported"); + // A default method is the interface's own code and is inherited as it is; + // it reaches the generated methods through `this`, as in Spring Data. + if (!method.isAbstract()) continue; try { methods.append(generator.method(method)); custom.add(method.getName()); } catch (IllegalArgumentException error) { throw invalid(method.getName() + ": " + error.getMessage()); } } @@ -86,7 +88,13 @@ static String generate(AnnotatedClass repository, Map findAll() { return session.query(" + e + ".class).list(); }"}, {"count", "public long count() { return session.query(" + e + ".class).count(); }"}, - {"existsById", "public boolean existsById(" + k + " id) { return session.find(" + e + ".class,id)!=null; }"} + {"existsById", "public boolean existsById(" + k + " id) { return session.find(" + e + ".class,id)!=null; }"}, + {"delete", "public void delete(" + e + " entity) { requireTransaction(); if(entity==null) throw new IllegalArgumentException(\"entity is null\"); if(session.contains(entity)) { session.remove(entity); return; } " + e + " value=session.merge(entity); if(value!=null) session.remove(value); }"}, + {"deleteAll", "public void deleteAll() { requireTransaction(); for(" + e + " value : session.query(" + e + ".class).list()) session.remove(value); }"}, + {"deleteAll", "public void deleteAll(Iterable entities) { requireTransaction(); if(entities==null) throw new IllegalArgumentException(\"entities is null\"); for(" + e + " value : entities) delete(value); }"}, + {"saveAll", "public java.util.List<" + e + "> saveAll(Iterable entities) { if(entities==null) throw new IllegalArgumentException(\"entities is null\"); java.util.List<" + e + "> out=new java.util.ArrayList<" + e + ">(); for(" + e + " value : entities) out.add(session.merge(value)); return out; }"}, + {"findAllById", "public java.util.List<" + e + "> findAllById(Iterable<" + k + "> ids) { if(ids==null) throw new IllegalArgumentException(\"ids is null\"); java.util.List<" + e + "> out=new java.util.ArrayList<" + e + ">(); for(" + k + " id : ids) { " + e + " value=session.find(" + e + ".class,id); if(value!=null) out.add(value); } return out; }"}, + {"flush", "public void flush() { session.flush(); }"} }; for (String[] method : defaults) if (!custom.contains(method[0])) methods.append(method[1]).append('\n'); int dot = repository.getBinaryName().lastIndexOf('.'); @@ -124,7 +132,9 @@ private String method(MethodInfo method) { return declaration(method, args, entityType) + "return session.find(" + entityType + ".class,p0,com.codename1.orm.session.LockMode." + mode[1] + ");}\n"; } if (name.equals("findById") || name.equals("save") || name.equals("insert") || name.equals("deleteById") - || name.equals("findAll") || name.equals("count") || name.equals("existsById")) { + || name.equals("findAll") || name.equals("count") || name.equals("existsById") + || name.equals("delete") || name.equals("deleteAll") || name.equals("saveAll") + || name.equals("findAllById") || name.equals("flush")) { throw invalid("Use the inherited CRUD method; only findById may be redeclared with @Lock"); } boolean list = result.getClassName().equals("java.util.List"); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java index 5be5093aee7..581be9a85cf 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -78,6 +78,7 @@ public final class RestControllerAnnotationProcessor extends AbstractAnnotationP private static final String MVC = "com.codename1.backend.mvc."; private static final String CONTROLLER = PKG + "RestController;"; private static final String REQUEST_MAPPING = PKG + "RequestMapping;"; + private static final String CROSS_ORIGIN = PKG + "CrossOrigin;"; private static final String PATH_VARIABLE = PKG + "PathVariable;"; private static final String REQUEST_PARAM = PKG + "RequestParam;"; private static final String REQUEST_HEADER = PKG + "RequestHeader;"; @@ -277,6 +278,12 @@ private static final class Route { String codecWrite; /** Whether any parameter is checked before the call. */ boolean validates; + /** The media types the route consumes, comma separated, or null for any. */ + String consumes; + /** The media types the route produces, comma separated, or null for any. */ + String produces; + /** The expression building the route's @CrossOrigin policy, or null. */ + String cors; } private static final class Param { @@ -408,6 +415,7 @@ public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws Proces } String httpMethod = null; List paths = null; + AnnotationValues used = null; for (Map.Entry e : MAPPINGS.entrySet()) { AnnotationValues values = m.getAnnotation(e.getKey()); if (values != null) { @@ -418,6 +426,7 @@ public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws Proces } httpMethod = e.getValue(); paths = pathsOf(values); + used = values; } } AnnotationValues mapping = m.getAnnotation(REQUEST_MAPPING); @@ -429,6 +438,7 @@ public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws Proces } httpMethod = mapping.getStringOrDefault("method", "GET"); paths = pathsOf(mapping); + used = mapping; } if (httpMethod == null) { continue; @@ -456,6 +466,14 @@ public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws Proces if (route == null) { return; } + // A method's own consumes/produces win; the class's apply to + // a method that names none, as in Spring. + AnnotationValues classMapping = cls.getClassAnnotation(REQUEST_MAPPING); + route.consumes = mediaTypes(used, classMapping, "consumes"); + route.produces = mediaTypes(used, classMapping, "produces"); + AnnotationValues crossOrigin = m.getAnnotation(CROSS_ORIGIN); + route.cors = crossOriginPolicy(crossOrigin != null ? crossOrigin + : cls.getClassAnnotation(CROSS_ORIGIN)); controller.routes.add(route); } } @@ -1424,6 +1442,72 @@ private static String joinPaths(String base, String path) { return joined.length() == 0 ? "/" : joined; } + /// The comma separated media types `name` lists on the method's mapping, or + /// on the class's when the method names none; null when neither does. + private static String mediaTypes(AnnotationValues method, AnnotationValues cls, String name) { + String own = joined(method, name); + return own != null ? own : joined(cls, name); + } + + private static String joined(AnnotationValues values, String name) { + List items = stringsOf(values, name); + if (items.isEmpty()) { + return null; + } + StringBuilder out = new StringBuilder(); + for (String item : items) { + if (out.length() > 0) { + out.append(','); + } + out.append(item.trim()); + } + return out.toString(); + } + + /// The expression that builds a `@CrossOrigin` policy, or null for none. + /// `value` is Spring's alias of `origins`, so it wins when given. + private static String crossOriginPolicy(AnnotationValues values) { + if (values == null) { + return null; + } + List origins = stringsOf(values, "value"); + if (origins.isEmpty()) { + origins = values.get("origins") == null ? Collections.singletonList("*") : stringsOf(values, "origins"); + } + List headers = values.get("allowedHeaders") == null ? Collections.singletonList("*") + : stringsOf(values, "allowedHeaders"); + return "com.codename1.backend.Cors.policy(" + stringArray(origins) + ", " + + stringArray(stringsOf(values, "methods")) + ", " + stringArray(headers) + ", " + + stringArray(stringsOf(values, "exposedHeaders")) + ", " + + values.getBoolOrDefault("allowCredentials", false) + ", " + + values.getIntOrDefault("maxAge", 1800) + ")"; + } + + private static String stringArray(List items) { + StringBuilder out = new StringBuilder("new String[] {"); + for (int i = 0; i < items.size(); i++) { + out.append(i > 0 ? ", " : "").append(quote(items.get(i))); + } + return out.append('}').toString(); + } + + private static List stringsOf(AnnotationValues values, String name) { + List out = new ArrayList(); + Object value = values == null ? null : values.get(name); + if (value instanceof List) { + for (Object item : (List) value) { + out.add(String.valueOf(item)); + } + } else if (value instanceof Object[]) { + for (Object item : (Object[]) value) { + out.add(String.valueOf(item)); + } + } else if (value != null) { + out.add(String.valueOf(value)); + } + return out; + } + private static List pathsOf(AnnotationValues values) { List out = new ArrayList(); if (values == null) { @@ -1673,10 +1757,15 @@ private static String generateRouter(Controller c) { .append(". Do not edit.\n"); sb.append("@com.codename1.backend.annotations.Generated\n"); boolean local = !c.handlers.isEmpty(); + boolean crossOrigin = false; + for (Route route : c.routes) { + crossOrigin |= route.cors != null; + } sb.append("public final class ").append(c.routerSimpleName) .append(" implements com.codename1.backend.HttpServer.Handler,\n") .append(" com.codename1.backend.web.ControllerHooks.Invocation") - .append(local ? ", com.codename1.backend.web.ExceptionResolver" : "").append(" {\n\n"); + .append(local ? ", com.codename1.backend.web.ExceptionResolver" : "") + .append(crossOrigin ? ", com.codename1.backend.web.CorsRoutes" : "").append(" {\n\n"); List ordered = new ArrayList(c.routes); Collections.sort(ordered, new java.util.Comparator() { @@ -1714,6 +1803,10 @@ public int compare(Route a, Route b) { sb.append(" private static final String[] A").append(i).append(" = ") .append(stringArrayLiteral(route.after)).append(";\n"); } + if (route.cors != null) { + sb.append(" private static final com.codename1.backend.Cors C").append(i).append(" = ") + .append(route.cors).append(";\n"); + } sb.append(" private static final com.codename1.backend.web.HandlerMethod H").append(i) .append(" = new com.codename1.backend.web.HandlerMethod(").append(quote(c.binaryName)) .append(", ").append(quote(route.javaMethod)).append(", ") @@ -1737,6 +1830,31 @@ public int compare(Route a, Route b) { // throws Exception because Handler does: a controller method that reads a // database throws IOException, and a router that could not pass it on would // force every handler to swallow its own errors. + if (crossOrigin) { + // The route a request is for -- for a preflight, the one its + // Access-Control-Request-Method asks about -- matched the way handle() + // matches it, so the server can apply that route's policy. + sb.append(" public com.codename1.backend.Cors corsPolicy(com.codename1.backend.HttpServer.Request request) {\n"); + sb.append(" String httpMethod = request.getMethod();\n"); + sb.append(" String asked = request.getHeader(\"Access-Control-Request-Method\");\n"); + sb.append(" if (\"OPTIONS\".equals(httpMethod) && asked != null) httpMethod = asked.trim();\n"); + for (int i = 0; i < ordered.size(); i++) { + Route route = ordered.get(i); + if (route.cors == null) { + continue; + } + sb.append(" if ((\"").append(route.httpMethod).append("\".equals(httpMethod)") + .append("GET".equals(route.httpMethod) ? " || \"HEAD\".equals(httpMethod)" : "").append(") && "); + if (route.after.isEmpty()) { + sb.append("request.pathIs(P").append(i).append(")"); + } else { + sb.append("request.pathStartsWith(P").append(i).append(") && bindPath(request.pathFrom(P") + .append(i).append(".length), A").append(i).append(") != null"); + } + sb.append(") return C").append(i).append(";\n"); + } + sb.append(" return null;\n }\n\n"); + } sb.append(" public com.codename1.backend.HttpServer.Response handle(\n") .append(" com.codename1.backend.HttpServer.Request request) throws Exception {\n"); sb.append(" String httpMethod = request.getMethod();\n"); @@ -1863,6 +1981,14 @@ private static void emitRoute(StringBuilder sb, Route route, int index, Controll sb.append(pad).append("com.codename1.backend.Tracing.route(") .append(quote(route.pattern)).append(");\n"); + if (route.consumes != null) { + sb.append(pad).append("if (!com.codename1.backend.web.MediaTypes.consumable(request.getHeader(\"Content-Type\"), ") + .append(quote(route.consumes)).append(")) return com.codename1.backend.HttpServer.Response.text(415, \"Unsupported Media Type\");\n"); + } + if (route.produces != null) { + sb.append(pad).append("if (!com.codename1.backend.web.MediaTypes.acceptable(request.getHeader(\"Accept\"), ") + .append(quote(route.produces)).append(")) return com.codename1.backend.HttpServer.Response.text(406, \"Not Acceptable\");\n"); + } emitPartGuards(sb, route, pad); emitRequiredGuards(sb, route, pad); emitScalarGuards(sb, route, pad); diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java index c30b32639c4..cfe47f889df 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java @@ -81,7 +81,9 @@ private void generatedRepository(boolean backend) throws Exception { + "@Query(\"select e.name, e.score from repositories.Note e where e.name=:name order by e.score desc\") java.util.Optional best(@Param(\"name\") String name); " + "@Modifying @Query(\"update repositories.Note e set e.score=:score where e.name=:name\") int revise(@Param(\"name\") String name,@Param(\"score\") long score); " + "@Modifying(clearAutomatically=true) @Query(\"update repositories.Note e set e.score=:score where e.name=:name\") void reviseAndClear(@Param(\"name\") String name,@Param(\"score\") long score); " - + "@Query(value=\"SELECT COUNT(*) FROM repository_note WHERE name=:name\",nativeSql=true) Long nativeCount(@Param(\"name\") String name); }"); + + "@Query(value=\"SELECT COUNT(*) FROM repository_note WHERE name=:name\",nativeSql=true) Long nativeCount(@Param(\"name\") String name); " + // A default method is the interface's own and reaches the generated ones through this. + + "default long countSame() { return countByName(\"same\"); } }"); // Compile the caller with the interface, before generating any implementation. sources.put("repositories.Check","package repositories; import com.codename1.orm.repository.*; import com.codename1.orm.session.*; public class Check { public static void run(Session s) { " + "Notes repo=Repositories.create(Notes.class,s); s.createTables(); s.beginTransaction(); Note a=new Note(); a.name=\"same\";a.score=3;a.active=true;a.archived=false;repo.insert(a);Note b=new Note();b.name=\"same\";b.score=9;repo.insert(b);s.commitTransaction(); " @@ -95,8 +97,15 @@ private void generatedRepository(boolean backend) throws Exception { + "s.beginTransaction(); a.score=4; if(repo.revise(\"same\",11)!=2 || !s.contains(a) || a.score!=4) throw new AssertionError(\"bulk\"); s.rollbackTransaction(); " + "if(s.find(Note.class,a.id).score!=3) throw new AssertionError(\"rollback\"); " + "s.beginTransaction(); try { repo.findById(a.id); throw new AssertionError(\"SQLite should refuse row lock\"); } catch(UnsupportedOperationException expected) {} s.rollbackTransaction(); " + + "if(repo.countSame()!=2) throw new AssertionError(\"default method\"); " + + "java.util.List byIds=repo.findAllById(java.util.Arrays.asList(b.id,-1L,a.id)); if(byIds.size()!=2 || byIds.get(0).id!=b.id || byIds.get(1).id!=a.id) throw new AssertionError(\"findAllById\"); " + + "s.beginTransaction(); Note d=new Note(); d.name=\"bulk\"; Note e=new Note(); e.name=\"bulk\"; repo.insert(d); repo.insert(e); repo.flush(); s.commitTransaction(); " + + "s.beginTransaction(); d.score=7; if(repo.saveAll(java.util.Arrays.asList(d,e)).size()!=2) throw new AssertionError(\"saveAll\"); repo.delete(e); s.commitTransaction(); " + + "if(repo.countByName(\"bulk\")!=1 || s.find(Note.class,d.id).score!=7) throw new AssertionError(\"delete\"); " + + "s.beginTransaction(); repo.deleteAll(java.util.Collections.singletonList(d)); s.commitTransaction(); if(repo.countByName(\"bulk\")!=0) throw new AssertionError(\"deleteAll(list)\"); " + "s.beginTransaction(); if(repo.save(a)==a) throw new AssertionError(\"merge returns managed copy\"); repo.deleteById(b.id); s.commitTransaction(); " + "s.beginTransaction(); if(repo.deleteByName(\"same\")!=1) throw new AssertionError(\"delete\");s.commitTransaction();if(repo.count()!=0) throw new AssertionError(); " + + "s.beginTransaction(); Note f=new Note(); f.name=\"gone\"; repo.insert(f); s.commitTransaction(); s.beginTransaction(); repo.deleteAll(); s.commitTransaction(); if(repo.count()!=0) throw new AssertionError(\"deleteAll\"); " // A @Modifying statement leaves the session as it is, as Spring Data does by // default: an entity read before it is still managed, and a change made to it // afterwards is written. clearAutomatically empties the session instead. @@ -118,6 +127,51 @@ private void generatedRepository(boolean backend) throws Exception { finally { session.close();if(server!=null)server.close();if(client!=null)client.close();loader.close(); } } + /// `@Column(insertable = false, updatable = false)` is a column only the + /// application's own statement writes: a compare-and-swap claim sets it, and + /// neither the insert nor a later save of the entity overwrites it. `@Lob` is + /// accepted on a string and changes nothing. + @Test + public void readOnlyColumnIsWrittenOnlyByItsOwnStatement() throws Exception { + readOnlyColumn(false); + readOnlyColumn(true); + } + private void readOnlyColumn(boolean backend) throws Exception { + File classes=tmp.newFolder(); + Map sources=new java.util.LinkedHashMap(); + sources.put("readonly.Job","package readonly; import com.codename1.annotations.*; import com.codename1.annotations.db.Lob; @Entity public class Job { @Id public long id; public String name; @Column(insertable=false,updatable=false) public String owner; @Lob public String body; }"); + sources.put("readonly.Jobs","package readonly; import com.codename1.orm.repository.*; @Repository(entity=Job.class,key=Long.class) public interface Jobs extends CrudRepository { " + + "@Modifying @Query(\"update readonly.Job e set e.owner=:owner where e.id=:id and e.owner is null\") int claim(@Param(\"id\") long id,@Param(\"owner\") String owner); }"); + sources.put("readonly.Check","package readonly; import com.codename1.orm.repository.*; import com.codename1.orm.session.*; public class Check { public static void run(Session s) { " + + "Jobs repo=Repositories.create(Jobs.class,s); s.createTables(); " + + "s.beginTransaction(); Job j=new Job(); j.name=\"a\"; j.owner=\"ignored\"; j.body=\"text\"; repo.insert(j); s.commitTransaction(); " + + "s.clear(); if(repo.findById(j.id).owner!=null) throw new AssertionError(\"insertable=false wrote the column\"); " + + "s.beginTransaction(); if(repo.claim(j.id,\"me\")!=1 || repo.claim(j.id,\"you\")!=0) throw new AssertionError(\"claim\"); s.commitTransaction(); " + + "s.clear(); Job loaded=repo.findById(j.id); if(!\"me\".equals(loaded.owner)) throw new AssertionError(\"claimed owner\"); " + + "s.beginTransaction(); loaded.owner=\"stolen\"; loaded.name=\"b\"; repo.save(loaded); s.commitTransaction(); " + + "s.clear(); Job after=repo.findById(j.id); if(!\"me\".equals(after.owner) || !\"b\".equals(after.name) || !\"text\".equals(after.body)) throw new AssertionError(\"updatable=false was overwritten: \"+after.owner+\" \"+after.name); " + + "} }"); + JavaSourceCompiler.compile(sources,classes,Arrays.asList(testClassesDir())); + ProcessorContext ctx=runProcessor(classes,backend ? backendClasspath() : Collections.emptyList());assertFalse(ctx.getErrors().toString(),ctx.hasErrors()); + java.net.URLClassLoader loader=new java.net.URLClassLoader(new URL[]{classes.toURI().toURL()},getClass().getClassLoader()); + loader.loadClass(backend ? "cn1app.BackendDaoBootstrap" : "cn1app.DaoBootstrap").newInstance(); + com.codename1.backend.orm.EntityManager server=backend ? com.codename1.backend.orm.EntityManager.open(com.codename1.backend.Database.open(":memory:")) : null; + com.codename1.orm.EntityManager client=backend ? null : com.codename1.orm.EntityManager.open(new com.codename1.impl.javase.SEDatabase(java.sql.DriverManager.getConnection("jdbc:sqlite::memory:"))); + com.codename1.orm.session.Session session=backend ? server.openSession() : client.openSession(); + try { loader.loadClass("readonly.Check").getMethod("run",com.codename1.orm.session.Session.class).invoke(null,session); } + finally { session.close();if(server!=null)server.close();if(client!=null)client.close();loader.close(); } + } + + @Test + public void lobOnANonStringFieldFailsTheBuild() throws Exception { + File classes=tmp.newFolder(); + Map sources=new java.util.LinkedHashMap(); + sources.put("badlob.Row","package badlob; import com.codename1.annotations.*; @Entity public class Row { @Id public long id; @com.codename1.annotations.db.Lob public long size; }"); + JavaSourceCompiler.compile(sources,classes,Arrays.asList(testClassesDir())); + ProcessorContext ctx=runProcessor(classes,backendClasspath()); + assertTrue(ctx.getErrors().toString(),ctx.getErrors().toString().contains("@Lob applies to")); + } + @Test public void nonPublicFieldsArePersistedOnBothRuntimes() throws Exception { nonPublicFields(false); diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java index 152cb7bea1f..5bd2b0165d2 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java @@ -178,6 +178,49 @@ public void bindsBodyAndStatus() throws Exception { assertEquals("{\"body\":\"hi\"}", Router.bodyOf(response)); } + /// `consumes` answers 415 to a request outside it, `produces` 406 to one that + /// accepts none of it; the class's apply where the method names none. + @Test + public void consumesAndProducesNegotiate() throws Exception { + Router router = generate("package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController @RequestMapping(value = \"/m\", produces = \"application/json\")\n" + + "public class Notes {\n" + + " @PostMapping(value = \"/in\", consumes = {\"application/json\", \"text/*\"})\n" + + " public String in(@RequestBody String body) { return \"ok\"; }\n" + + " @GetMapping(value = \"/out\")\n" + + " public String out() { return \"ok\"; }\n" + + "}\n"); + assertEquals(415, Router.statusOf(router.callWith("POST", "/m/in", "x", "Content-Type: application/xml"))); + assertEquals(415, Router.statusOf(router.callWith("POST", "/m/in", "x"))); + assertEquals(200, Router.statusOf(router.callWith("POST", "/m/in", "x", "Content-Type: Application/JSON; charset=utf-8"))); + assertEquals(200, Router.statusOf(router.callWith("POST", "/m/in", "x", "Content-Type: text/plain"))); + assertEquals(406, Router.statusOf(router.callWith("GET", "/m/out", null, "Accept: text/html"))); + assertEquals(406, Router.statusOf(router.callWith("GET", "/m/out", null, "Accept: application/json;q=0, text/html"))); + assertEquals(200, Router.statusOf(router.callWith("GET", "/m/out", null, "Accept: text/html, */*;q=0.1"))); + assertEquals(200, Router.statusOf(router.callWith("GET", "/m/out", null))); + } + + /// A `@CrossOrigin` route reports its own policy, the preflight included, and + /// a route without one reports none so the global policy applies. + @Test + public void crossOriginNamesTheRoutesPolicy() throws Exception { + Router router = generate("package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController @RequestMapping(\"/c\")\n" + + "public class Notes {\n" + + " @CrossOrigin(origins = \"https://app.example.com\", allowCredentials = true)\n" + + " @GetMapping(\"/items/{id}\") public String item(@PathVariable(\"id\") String id) { return id; }\n" + + " @GetMapping(\"/plain\") public String plain() { return \"p\"; }\n" + + "}\n"); + assertNotNull(router.corsPolicy("GET", "/c/items/7")); + assertNotNull("a preflight is matched by the method it asks about", + router.corsPolicy("OPTIONS", "/c/items/7", "Origin: https://app.example.com", + "Access-Control-Request-Method: GET")); + assertNull(router.corsPolicy("POST", "/c/items/7")); + assertNull(router.corsPolicy("GET", "/c/plain")); + } + @Test public void doesNotMatchWhatItShouldNot() throws Exception { Router router = generate(CONTROLLER_SOURCE); @@ -2044,6 +2087,16 @@ Object call(String method, String target, String body) throws Exception { return handle.invoke(instance, request(method, target, body)); } + /// A request carrying `headers`, each "Name: value". + Object callWith(String method, String target, String body, String... headers) throws Exception { + return handle.invoke(instance, request(method, target, body, headers)); + } + + Object corsPolicy(String method, String target, String... headers) throws Exception { + return instance.getClass().getMethod("corsPolicy", HttpServer.Request.class) + .invoke(instance, request(method, target, null, headers)); + } + String text2(String method, String target, String body) throws Exception { Object response = call(method, target, body); assertNotNull(method + " " + target + " matched no route", response); @@ -2071,13 +2124,25 @@ static String bodyOf(Object response) throws Exception { return new String((byte[]) bodyField.get(response), "UTF-8"); } - private static HttpServer.Request request(String method, String target, String body) - throws Exception { + private static HttpServer.Request request(String method, String target, String body, + String... headers) throws Exception { reflect(); - byte[] raw = (method + " " + target + " HTTP/1.1\r\nUser-Agent: probe\r\n\r\n") - .getBytes("UTF-8"); + StringBuilder text = new StringBuilder(method + " " + target + " HTTP/1.1\r\n"); + int[] slices = new int[headers.length * 4]; + for (int i = 0; i < headers.length; i++) { + int colon = headers[i].indexOf(':'); + String name = headers[i].substring(0, colon); + String value = headers[i].substring(colon + 1).trim(); + slices[i * 4] = text.length(); + slices[i * 4 + 1] = name.length(); + text.append(name).append(": "); + slices[i * 4 + 2] = text.length(); + slices[i * 4 + 3] = value.length(); + text.append(value).append("\r\n"); + } + byte[] raw = text.append("\r\n").toString().getBytes("UTF-8"); return (HttpServer.Request) requestCtor.newInstance(method, target, "HTTP/1.1", raw, - new int[0], 0, body, Integer.valueOf(method.length() + 1), + slices, Integer.valueOf(headers.length), body, Integer.valueOf(method.length() + 1), Integer.valueOf(target.getBytes("UTF-8").length)); } diff --git a/vm/backend/src/com/codename1/backend/Backend.java b/vm/backend/src/com/codename1/backend/Backend.java index cc3b17b3851..e4fdd2c27f9 100644 --- a/vm/backend/src/com/codename1/backend/Backend.java +++ b/vm/backend/src/com/codename1/backend/Backend.java @@ -900,6 +900,7 @@ public ForwardedHeaders forwardedHeaders() { /// gets, as Spring's filter puts them on every response. @Override public void decorateFallback(HttpServer.Request request, HttpServer.Response response) { + Cors cors = corsFor(request); if (cors != null) { cors.decorate(request, response); } @@ -912,6 +913,24 @@ public void decorateFallback(HttpServer.Request request, HttpServer.Response res /// The first of the application's routers to take `request`, or null. /// What the security layer calls once it lets a request through. + /// The CORS policy that governs `request`: the `@CrossOrigin` of the route + /// it is for (the route a preflight asks about), which replaces the global + /// one for that route, as a handler-level policy is the more specific; or + /// the global `cn1.cors.*` policy, or null when neither applies. Consulted + /// for the reject, the preflight and the response headers alike, so the + /// three never disagree about which policy a request is under. + Cors corsFor(HttpServer.Request request) { + for (int i = 0 ; i < chain.length ; i++) { + if (chain[i] instanceof com.codename1.backend.web.CorsRoutes) { + Cors own = ((com.codename1.backend.web.CorsRoutes) chain[i]).corsPolicy(request); + if (own != null) { + return own; + } + } + } + return cors; + } + @Override public HttpServer.Response route(HttpServer.Request request) throws Exception { HttpServer.Response response = null; @@ -1114,6 +1133,7 @@ public HttpServer.Response handle(HttpServer.Request request) try { HttpServer.Response response = null; List attempted = new ArrayList(2); + Cors cors = corsFor(request); try { // A cross-origin request the CORS policy does not admit never // reaches a handler. @@ -2226,6 +2246,7 @@ public void register(HttpServer.WebSocketRegistry direct) JsonCodec.MAX_DEPTH_SETTING, JsonCodec.MAX_DEPTH)); Cors cors = Cors.fromConfig(config); + Cors.routesServedOverTls(context != null); if (cors != null) { // Whether a request arrives over TLS, for telling a same-origin // request from a cross-origin one: the scheme is part of an origin. diff --git a/vm/backend/src/com/codename1/backend/Cors.java b/vm/backend/src/com/codename1/backend/Cors.java index 6c6aecab86c..5f6777ceae4 100644 --- a/vm/backend/src/com/codename1/backend/Cors.java +++ b/vm/backend/src/com/codename1/backend/Cors.java @@ -37,7 +37,7 @@ /// itself, so an endpoint with an origin policy of its own -- the MCP endpoint -- /// keeps it. An ordinary response gains the headers only when its handler set no /// `Access-Control-Allow-Origin` of its own. -final class Cors { +public final class Cors { static final String ALLOWED_ORIGINS = "cn1.cors.allowedOrigins"; static final String ALLOWED_METHODS = "cn1.cors.allowedMethods"; static final String ALLOWED_HEADERS = "cn1.cors.allowedHeaders"; @@ -67,6 +67,20 @@ private Cors(List origins, String methods, String headers, String exposed, /// Whether the server speaks TLS; set once, before it serves anything. private boolean tls; + /// Whether this is a `@CrossOrigin` route's policy. Those are static objects + /// of the generated router, made before the server starts, so they read the + /// scheme from [#ROUTES_TLS] rather than from a field the server sets. + private boolean route; + + /// Whether the running server speaks TLS, for route policies. + private static final java.util.concurrent.atomic.AtomicBoolean ROUTES_TLS = + new java.util.concurrent.atomic.AtomicBoolean(); + + /// Records, for every route policy, whether the server speaks TLS. + static void routesServedOverTls(boolean value) { + ROUTES_TLS.set(value); + } + /// Records whether requests arrive over TLS, which decides the scheme of the /// server's own origin. void servedOverTls(boolean value) { @@ -116,6 +130,48 @@ static Cors fromConfig(Config config) throws IOException { config.getInt(MAX_AGE, 1800)); } + /// The policy of a `@CrossOrigin` route, built once by the generated router. + /// An empty `methods` allows every method, since the route already decides + /// which one it answers; `origins` follow the same rules as + /// `cn1.cors.allowedOrigins`. + /// + /// @throws IllegalArgumentException for an origin that is not + /// scheme://host[:port], or credentials allowed with `*` + public static Cors policy(String[] origins, String[] methods, String[] headers, String[] exposed, + boolean credentials, int maxAge) { + List listed = new ArrayList(); + for (int i = 0; i < origins.length; i++) { + String origin = origins[i].trim(); + if ("*".equals(origin)) { + listed.add(origin); + } else if (isOrigin(origin)) { + listed.add(canonicalOrigin(origin)); + } else { + throw new IllegalArgumentException("@CrossOrigin names \"" + origin + + "\"; an origin is scheme://host[:port], with no path"); + } + } + if (credentials && listed.contains("*")) { + throw new IllegalArgumentException("@CrossOrigin(allowCredentials = true) needs the origins " + + "listed; a browser refuses credentials with *"); + } + Cors policy = new Cors(listed, methods.length == 0 ? "*" : join(methods), join(headers), + join(exposed), credentials, maxAge); + policy.route = true; + return policy; + } + + private static String join(String[] values) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + if (i > 0) { + out.append(", "); + } + out.append(values[i].trim()); + } + return out.toString(); + } + /// Whether `origin` may read responses. boolean allows(String origin) { if (origin == null) { @@ -197,7 +253,7 @@ private String requestScheme(HttpServer.Request request) { return first; } } - return tls ? "https" : "http"; + return (route ? ROUTES_TLS.get() : tls) ? "https" : "http"; } /// Whether `origin` names the server the request was sent to: the same scheme, diff --git a/vm/backend/src/com/codename1/backend/ResponseStatusException.java b/vm/backend/src/com/codename1/backend/ResponseStatusException.java index 695dfced80c..0c727ab111e 100644 --- a/vm/backend/src/com/codename1/backend/ResponseStatusException.java +++ b/vm/backend/src/com/codename1/backend/ResponseStatusException.java @@ -49,6 +49,14 @@ public class ResponseStatusException extends RuntimeException { private final int status; private final String reason; + /// Answers with the status alone and an empty body, as Spring's + /// `new ResponseStatusException(HttpStatus.NOT_FOUND)` does. + /// + /// @param status the status to answer with, 400 to 599 + public ResponseStatusException(int status) { + this(status, null, null); + } + /// @param status the status to answer with, 400 to 599 /// @param reason what to tell the caller; sent as the body public ResponseStatusException(int status, String reason) { @@ -59,7 +67,7 @@ public ResponseStatusException(int status, String reason) { /// @param reason what to tell the caller; sent as the body /// @param cause what led to it, for the log; never sent public ResponseStatusException(int status, String reason, Throwable cause) { - super(status + " " + reason, cause); + super(reason == null ? String.valueOf(status) : status + " " + reason, cause); if (status < 400 || status > 599) { throw new IllegalArgumentException("not an error status: " + status); } diff --git a/vm/backend/src/com/codename1/backend/annotations/CrossOrigin.java b/vm/backend/src/com/codename1/backend/annotations/CrossOrigin.java new file mode 100644 index 00000000000..e1df37b8954 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/CrossOrigin.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// A cross-origin policy for one controller or one handler, as Spring's +/// `@CrossOrigin` is. On a class it covers every route of the controller; on a +/// method it replaces the class's for that route. +/// +/// The route's policy replaces the global `cn1.cors.*` policy for requests to +/// that route -- the preflight, the reject of an origin it does not list and the +/// response headers alike -- rather than being merged with it, so what a route +/// allows is read off its annotation alone. A preflight for such a route is +/// answered before the security chain, as the global policy's is, because a +/// browser sends no credentials with one. +/// +/// ```java +/// @CrossOrigin(origins = "https://app.example.com", allowCredentials = true) +/// @GetMapping("/api/me") +/// public Profile me(@AuthenticationPrincipal User user) { ... } +/// ``` +@Retention(RetentionPolicy.CLASS) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface CrossOrigin { + /// The origins allowed, each `scheme://host[:port]`, or `*` for any. + String[] origins() default {"*"}; + /// The same as [#origins], as Spring spells it on its `value` attribute. + String[] value() default {}; + /// The methods a preflight may ask for; empty allows the route's own. + String[] methods() default {}; + /// The request headers a preflight may ask for; `*` allows any. + String[] allowedHeaders() default {"*"}; + /// Response headers a script may read beyond the safelisted ones. + String[] exposedHeaders() default {}; + /// Whether the browser may send cookies and read the response with them. + /// Needs [#origins] to list the origins: a browser refuses `*` with credentials. + boolean allowCredentials() default false; + /// How long, in seconds, a browser may cache the preflight answer. + int maxAge() default 1800; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java b/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java index 563b1930853..62a3f461daf 100644 --- a/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java @@ -35,4 +35,11 @@ public @interface DeleteMapping { /// The path, relative to any [RequestMapping] on the class. String[] value() default {}; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/annotations/GetMapping.java b/vm/backend/src/com/codename1/backend/annotations/GetMapping.java index 50166fd83f5..5359d0b4eb1 100644 --- a/vm/backend/src/com/codename1/backend/annotations/GetMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/GetMapping.java @@ -35,4 +35,11 @@ public @interface GetMapping { /// The path, relative to any [RequestMapping] on the class. String[] value() default {}; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java b/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java index 39fe4c78fa7..c1958cc0b4a 100644 --- a/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java @@ -35,4 +35,11 @@ public @interface PatchMapping { /// The path, relative to any [RequestMapping] on the class. String[] value() default {}; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/annotations/PostMapping.java b/vm/backend/src/com/codename1/backend/annotations/PostMapping.java index 2fe2db7fac4..0d533821329 100644 --- a/vm/backend/src/com/codename1/backend/annotations/PostMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/PostMapping.java @@ -35,4 +35,11 @@ public @interface PostMapping { /// The path, relative to any [RequestMapping] on the class. String[] value() default {}; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/annotations/PutMapping.java b/vm/backend/src/com/codename1/backend/annotations/PutMapping.java index 1f02608c0bd..d8b4e6dfcfa 100644 --- a/vm/backend/src/com/codename1/backend/annotations/PutMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/PutMapping.java @@ -35,4 +35,11 @@ public @interface PutMapping { /// The path, relative to any [RequestMapping] on the class. String[] value() default {}; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java b/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java index 8aeef9d4f21..d923241cf56 100644 --- a/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java +++ b/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java @@ -38,4 +38,13 @@ String[] value() default {}; /// The HTTP verb, when this is used on a method directly. String method() default ""; + /// Media types the route takes, matched against the request's + /// `Content-Type`; a request outside them is answered 415. Empty takes any. + /// On a class it applies to every route that names none of its own. + String[] consumes() default {}; + /// Media types the route answers with, matched against the request's + /// `Accept`; a request that accepts none of them is answered 406. Empty + /// answers anything. The handler still sets the type of what it returns. + /// On a class it applies to every route that names none of its own. + String[] produces() default {}; } diff --git a/vm/backend/src/com/codename1/backend/web/CorsRoutes.java b/vm/backend/src/com/codename1/backend/web/CorsRoutes.java new file mode 100644 index 00000000000..f8f1578ed9c --- /dev/null +++ b/vm/backend/src/com/codename1/backend/web/CorsRoutes.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.web; + +import com.codename1.backend.Cors; +import com.codename1.backend.HttpServer; + +/// Implemented by a generated router whose controllers carry `@CrossOrigin`: +/// answers the policy of the route a request is for, so the server applies it +/// in place of the global `cn1.cors.*` one. +/// +/// Internal to generated code; an application does not implement it. +public interface CorsRoutes { + /// The `@CrossOrigin` policy of the route `request` is for -- for a preflight, + /// the route its `Access-Control-Request-Method` asks about -- or null when + /// that route has none. + Cors corsPolicy(HttpServer.Request request); +} diff --git a/vm/backend/src/com/codename1/backend/web/MediaTypes.java b/vm/backend/src/com/codename1/backend/web/MediaTypes.java new file mode 100644 index 00000000000..c166d8f22d3 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/web/MediaTypes.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.web; + +/// Media-type matching for a route's `consumes` and `produces`, called by the +/// generated router before it binds anything. +/// +/// Type names are compared without case folding through `toLowerCase`, which is +/// locale sensitive; parameters (`; charset=utf-8`) are ignored, and either side +/// may use a `*` wildcard for the type or the subtype. +/// +/// Internal to generated code. +public final class MediaTypes { + private MediaTypes() { + } + + /// Whether a request whose `Content-Type` is `contentType` may reach a route + /// that consumes `accepted`, a comma separated list. A request with no + /// `Content-Type` is not consumable: it is answered 415, as Spring answers it. + public static boolean consumable(String contentType, String accepted) { + if (contentType == null) { + return false; + } + String type = bare(contentType, 0, contentType.length()); + if (type.length() == 0) { + return false; + } + int pos = 0; + while (pos <= accepted.length()) { + int comma = accepted.indexOf(',', pos); + int end = comma < 0 ? accepted.length() : comma; + if (matches(bare(accepted, pos, end), type)) { + return true; + } + if (comma < 0) { + break; + } + pos = comma + 1; + } + return false; + } + + /// Whether a request whose `Accept` header is `accept` takes any of + /// `offered`, a comma separated list; when it does not, the route answers 406. + /// No `Accept` header accepts anything, and a range with `q=0` accepts nothing. + public static boolean acceptable(String accept, String offered) { + if (accept == null || accept.trim().length() == 0) { + return true; + } + int pos = 0; + while (pos <= accept.length()) { + int comma = accept.indexOf(',', pos); + int end = comma < 0 ? accept.length() : comma; + if (!refused(accept, pos, end)) { + String range = bare(accept, pos, end); + int o = 0; + while (range.length() > 0 && o <= offered.length()) { + int oc = offered.indexOf(',', o); + int oe = oc < 0 ? offered.length() : oc; + if (matches(range, bare(offered, o, oe))) { + return true; + } + if (oc < 0) { + break; + } + o = oc + 1; + } + } + if (comma < 0) { + break; + } + pos = comma + 1; + } + return false; + } + + /// The media type in `value[from, to)`, without parameters or whitespace. + private static String bare(String value, int from, int to) { + int semi = value.indexOf(';', from); + if (semi >= 0 && semi < to) { + to = semi; + } + return value.substring(from, to).trim(); + } + + /// Whether the range in `value[from, to)` carries `q=0`, which excludes it. + private static boolean refused(String value, int from, int to) { + int semi = value.indexOf(';', from); + while (semi >= 0 && semi < to) { + int next = value.indexOf(';', semi + 1); + int end = next < 0 || next > to ? to : next; + String param = value.substring(semi + 1, end).trim(); + if (param.length() > 2 && (param.charAt(0) == 'q' || param.charAt(0) == 'Q') && param.charAt(1) == '=') { + String q = param.substring(2).trim(); + boolean zero = q.length() > 0; + for (int i = 0; i < q.length(); i++) { + char c = q.charAt(i); + if (c != '0' && c != '.') { + zero = false; + } + } + return zero; + } + semi = next < 0 || next > to ? -1 : next; + } + return false; + } + + /// Whether two media types match, a `*` on either side matching anything in + /// its place. + private static boolean matches(String a, String b) { + int sa = a.indexOf('/'); + int sb = b.indexOf('/'); + if (sa < 0 || sb < 0) { + return "*".equals(a) || "*".equals(b); + } + return part(a, 0, sa, b, 0, sb) && part(a, sa + 1, a.length(), b, sb + 1, b.length()); + } + + private static boolean part(String a, int af, int at, String b, int bf, int bt) { + if (at - af == 1 && a.charAt(af) == '*' || bt - bf == 1 && b.charAt(bf) == '*') { + return true; + } + return at - af == bt - bf && a.regionMatches(true, af, b, bf, at - af); + } +} From 5ea06a93c9331fbb4fad068dc808160107d31010 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:56:55 +0200 Subject: [PATCH 02/25] JavaAPI: Semaphore, ConcurrentLinkedQueue, Flow, YearMonth, TemporalAdjusters, BigDecimal, InetAddress (#6153) Item 5 of #6153: classes the backend needed that the ParparVM class library lacked. BigDecimal keeps its unscaled value as decimal digits, since there is no java.math.BigInteger here. InetAddress resolves address literals only; the backend's getaddrinfo lives in its own TCP natives, so other host names throw UnknownHostException. LocalDate gains with(TemporalAdjuster) and implements TemporalAdjuster. JdkGapChecks holds all of them to HotSpot in the bulk conformance run. Co-Authored-By: Claude Opus 5.5 --- vm/JavaAPI/src/java/math/BigDecimal.java | 756 ++++++++++++++++++ vm/JavaAPI/src/java/math/RoundingMode.java | 38 + vm/JavaAPI/src/java/net/Inet4Address.java | 68 ++ vm/JavaAPI/src/java/net/Inet6Address.java | 88 ++ vm/JavaAPI/src/java/net/InetAddress.java | 281 +++++++ .../src/java/net/UnknownHostException.java | 34 + vm/JavaAPI/src/java/time/LocalDate.java | 25 +- vm/JavaAPI/src/java/time/YearMonth.java | 243 ++++++ .../java/time/temporal/TemporalAdjuster.java | 30 + .../java/time/temporal/TemporalAdjusters.java | 150 ++++ .../concurrent/ConcurrentLinkedQueue.java | 179 +++++ vm/JavaAPI/src/java/util/concurrent/Flow.java | 64 ++ .../src/java/util/concurrent/Semaphore.java | 178 +++++ .../tools/translator/bulk/JdkGapChecks.java | 284 +++++++ 14 files changed, 2417 insertions(+), 1 deletion(-) create mode 100644 vm/JavaAPI/src/java/math/BigDecimal.java create mode 100644 vm/JavaAPI/src/java/math/RoundingMode.java create mode 100644 vm/JavaAPI/src/java/net/Inet4Address.java create mode 100644 vm/JavaAPI/src/java/net/Inet6Address.java create mode 100644 vm/JavaAPI/src/java/net/InetAddress.java create mode 100644 vm/JavaAPI/src/java/net/UnknownHostException.java create mode 100644 vm/JavaAPI/src/java/time/YearMonth.java create mode 100644 vm/JavaAPI/src/java/time/temporal/TemporalAdjuster.java create mode 100644 vm/JavaAPI/src/java/time/temporal/TemporalAdjusters.java create mode 100644 vm/JavaAPI/src/java/util/concurrent/ConcurrentLinkedQueue.java create mode 100644 vm/JavaAPI/src/java/util/concurrent/Flow.java create mode 100644 vm/JavaAPI/src/java/util/concurrent/Semaphore.java create mode 100644 vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java diff --git a/vm/JavaAPI/src/java/math/BigDecimal.java b/vm/JavaAPI/src/java/math/BigDecimal.java new file mode 100644 index 00000000000..452e908df1b --- /dev/null +++ b/vm/JavaAPI/src/java/math/BigDecimal.java @@ -0,0 +1,756 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.math; + +/// `java.math.BigDecimal`: an exact decimal number, an arbitrary precision +/// unscaled integer and a scale, the value being `unscaled * 10^-scale`. +/// +/// This class library has no `java.math.BigInteger`, so the unscaled value +/// is kept as its decimal digits and the arithmetic is schoolbook decimal +/// arithmetic. That is exact and simple, and slow on numbers of hundreds of +/// digits -- this is meant for money and measurements, not for number +/// theory. For the same reason there is no `unscaledValue()`, +/// `toBigInteger()` or `MathContext` here. +public class BigDecimal extends Number implements Comparable { + + public static final BigDecimal ZERO = new BigDecimal(0, "0", 0); + public static final BigDecimal ONE = new BigDecimal(1, "1", 0); + public static final BigDecimal TEN = new BigDecimal(1, "10", 0); + + public static final int ROUND_UP = 0; + public static final int ROUND_DOWN = 1; + public static final int ROUND_CEILING = 2; + public static final int ROUND_FLOOR = 3; + public static final int ROUND_HALF_UP = 4; + public static final int ROUND_HALF_DOWN = 5; + public static final int ROUND_HALF_EVEN = 6; + public static final int ROUND_UNNECESSARY = 7; + + /// -1, 0 or 1. + private final int signum; + /// The unscaled magnitude: decimal digits, no leading zero, "0" for zero. + private final String mag; + private final int scale; + + private BigDecimal(int signum, String mag, int scale) { + this.signum = "0".equals(mag) ? 0 : signum; + this.mag = mag; + this.scale = scale; + } + + /// Parses `[sign] digits [. digits] [e|E [sign] digits]`. + public BigDecimal(String val) { + int len = val.length(); + if (len == 0) { + throw new NumberFormatException("Empty string"); + } + int i = 0; + int sign = 1; + char c = val.charAt(0); + if (c == '-' || c == '+') { + sign = c == '-' ? -1 : 1; + i++; + } + StringBuilder digits = new StringBuilder(); + int fraction = 0; + boolean dot = false; + boolean any = false; + long exp = 0; + for (; i < len; i++) { + c = val.charAt(i); + if (c >= '0' && c <= '9') { + digits.append(c); + any = true; + if (dot) { + fraction++; + } + } else if (c == '.' && !dot) { + dot = true; + } else if (c == 'e' || c == 'E') { + i++; + if (i >= len) { + throw new NumberFormatException("No exponent digits"); + } + boolean neg = false; + c = val.charAt(i); + if (c == '-' || c == '+') { + neg = c == '-'; + i++; + } + if (i >= len) { + throw new NumberFormatException("No exponent digits"); + } + for (; i < len; i++) { + c = val.charAt(i); + if (c < '0' || c > '9') { + throw new NumberFormatException("Bad exponent: " + val); + } + exp = exp * 10 + (c - '0'); + if (exp > Integer.MAX_VALUE) { + throw new NumberFormatException("Exponent overflow"); + } + } + if (neg) { + exp = -exp; + } + break; + } else { + throw new NumberFormatException("Character " + c + " is neither a decimal digit number," + + " decimal point, nor \"e\" notation exponential mark."); + } + } + if (!any) { + throw new NumberFormatException("No digits: " + val); + } + long s = fraction - exp; + if (s > Integer.MAX_VALUE || s < Integer.MIN_VALUE) { + throw new NumberFormatException("Scale out of range"); + } + String m = trim(digits.toString()); + this.signum = "0".equals(m) ? 0 : sign; + this.mag = m; + this.scale = (int) s; + } + + public BigDecimal(char[] in) { + this(new String(in)); + } + + public BigDecimal(int val) { + this((long) val); + } + + public BigDecimal(long val) { + this.signum = val > 0 ? 1 : (val < 0 ? -1 : 0); + this.mag = magOf(val); + this.scale = 0; + } + + /// The exact binary value of `val`, as the JDK's constructor gives it: + /// `new BigDecimal(0.1)` is 0.1000000000000000055511151231257827... + public BigDecimal(double val) { + if (Double.isNaN(val) || Double.isInfinite(val)) { + throw new NumberFormatException("Infinite or NaN"); + } + long bits = Double.doubleToLongBits(val); + int sign = (bits >> 63) == 0 ? 1 : -1; + int exponent = (int) ((bits >> 52) & 0x7ffL); + long significand = exponent == 0 ? (bits & ((1L << 52) - 1)) << 1 + : (bits & ((1L << 52) - 1)) | (1L << 52); + exponent -= 1075; + if (significand == 0) { + this.signum = 0; + this.mag = "0"; + this.scale = 0; + return; + } + while ((significand & 1) == 0) { + significand >>= 1; + exponent++; + } + String m = magOf(significand); + int s = 0; + if (exponent >= 0) { + m = mul(m, pow(2, exponent)); + } else { + // m / 2^k == m * 5^k / 10^k + m = mul(m, pow(5, -exponent)); + s = -exponent; + } + this.signum = sign; + this.mag = m; + this.scale = s; + } + + public static BigDecimal valueOf(long val) { + if (val == 0) { + return ZERO; + } + return new BigDecimal(val); + } + + public static BigDecimal valueOf(long unscaledVal, int scale) { + BigDecimal v = new BigDecimal(unscaledVal); + return new BigDecimal(v.signum, v.mag, scale); + } + + /// The value `Double.toString(val)` spells, as the JDK does. + public static BigDecimal valueOf(double val) { + return new BigDecimal(Double.toString(val)); + } + + public int signum() { + return signum; + } + + public int scale() { + return scale; + } + + /// The number of digits in the unscaled value. + public int precision() { + return mag.length(); + } + + public BigDecimal negate() { + return new BigDecimal(-signum, mag, scale); + } + + public BigDecimal abs() { + return signum < 0 ? negate() : this; + } + + public BigDecimal plus() { + return this; + } + + public BigDecimal add(BigDecimal augend) { + int s = Math.max(scale, augend.scale); + String a = shift(mag, s - scale); + String b = shift(augend.mag, s - augend.scale); + return addSigned(signum, a, augend.signum, b, s); + } + + public BigDecimal subtract(BigDecimal subtrahend) { + return add(subtrahend.negate()); + } + + public BigDecimal multiply(BigDecimal multiplicand) { + return new BigDecimal(signum * multiplicand.signum, mul(mag, multiplicand.mag), + checkScale((long) scale + multiplicand.scale)); + } + + /// The exact quotient; throws `ArithmeticException` when its decimal + /// expansion does not terminate. The scale is the smallest that holds + /// the result, but not below `this.scale() - divisor.scale()`. + public BigDecimal divide(BigDecimal divisor) { + if (divisor.signum == 0) { + throw new ArithmeticException(signum == 0 ? "Division undefined" : "Division by zero"); + } + int preferred = checkScale((long) scale - divisor.scale); + if (signum == 0) { + return new BigDecimal(0, "0", preferred); + } + // A terminating quotient a/b needs at most as many extra digits as + // the largest power of 2 or 5 dividing b, under 4 per digit of b. + int extra = 4 * divisor.mag.length() + 4; + String[] qr = divmod(shift(mag, extra), divisor.mag); + if (!"0".equals(qr[1])) { + throw new ArithmeticException("Non-terminating decimal expansion; no exact representable decimal result."); + } + String q = qr[0]; + int s = checkScale((long) scale - divisor.scale + extra); + while (s > preferred && q.length() > 1 && q.charAt(q.length() - 1) == '0') { + q = q.substring(0, q.length() - 1); + s--; + } + return new BigDecimal(signum * divisor.signum, q, s); + } + + public BigDecimal divide(BigDecimal divisor, RoundingMode roundingMode) { + return divide(divisor, scale, roundingMode); + } + + public BigDecimal divide(BigDecimal divisor, int scale, RoundingMode roundingMode) { + if (divisor.signum == 0) { + throw new ArithmeticException(signum == 0 ? "Division undefined" : "Division by zero"); + } + // unscaled result = a * 10^(scale - this.scale + divisor.scale) / b + long k = (long) scale - this.scale + divisor.scale; + String a = mag; + String b = divisor.mag; + if (k >= 0) { + a = shift(a, checkScale(k)); + } else { + b = shift(b, checkScale(-k)); + } + String[] qr = divmod(a, b); + int sign = signum * divisor.signum; + return new BigDecimal(sign, round(sign, qr[0], qr[1], b, roundingMode), scale); + } + + public BigDecimal divide(BigDecimal divisor, int scale, int roundingMode) { + return divide(divisor, scale, RoundingMode.valueOf(roundingMode)); + } + + public BigDecimal divide(BigDecimal divisor, int roundingMode) { + return divide(divisor, scale, RoundingMode.valueOf(roundingMode)); + } + + /// The integer part of `this / divisor`, at the preferred scale. + public BigDecimal divideToIntegralValue(BigDecimal divisor) { + BigDecimal q = divide(divisor, 0, RoundingMode.DOWN); + int preferred = checkScale((long) scale - divisor.scale); + return preferred > 0 ? q.setScale(preferred) : q; + } + + /// `this - this.divideToIntegralValue(divisor) * divisor`. + public BigDecimal remainder(BigDecimal divisor) { + return subtract(divide(divisor, 0, RoundingMode.DOWN).multiply(divisor)); + } + + public BigDecimal setScale(int newScale, RoundingMode roundingMode) { + if (newScale == scale) { + return this; + } + if (newScale > scale) { + return new BigDecimal(signum, shift(mag, newScale - scale), newScale); + } + String div = shift("1", checkScale((long) scale - newScale)); + String[] qr = divmod(mag, div); + return new BigDecimal(signum, round(signum, qr[0], qr[1], div, roundingMode), newScale); + } + + public BigDecimal setScale(int newScale, int roundingMode) { + return setScale(newScale, RoundingMode.valueOf(roundingMode)); + } + + /// The same value at `newScale`; throws `ArithmeticException` when that + /// would need rounding. + public BigDecimal setScale(int newScale) { + return setScale(newScale, RoundingMode.UNNECESSARY); + } + + public BigDecimal movePointLeft(int n) { + BigDecimal r = new BigDecimal(signum, mag, checkScale((long) scale + n)); + return r.scale < 0 ? r.setScale(0) : r; + } + + public BigDecimal movePointRight(int n) { + BigDecimal r = new BigDecimal(signum, mag, checkScale((long) scale - n)); + return r.scale < 0 ? r.setScale(0) : r; + } + + public BigDecimal scaleByPowerOfTen(int n) { + return new BigDecimal(signum, mag, checkScale((long) scale - n)); + } + + /// The same value with no trailing zeros in the unscaled value; zero is + /// [#ZERO]. + public BigDecimal stripTrailingZeros() { + if (signum == 0) { + return ZERO; + } + int end = mag.length(); + while (end > 1 && mag.charAt(end - 1) == '0') { + end--; + } + int removed = mag.length() - end; + return removed == 0 ? this : new BigDecimal(signum, mag.substring(0, end), checkScale((long) scale - removed)); + } + + public BigDecimal pow(int n) { + if (n < 0 || n > 999999999) { + throw new ArithmeticException("Invalid operation"); + } + BigDecimal r = ONE; + for (int i = 0; i < n; i++) { + r = r.multiply(this); + } + return r; + } + + public BigDecimal min(BigDecimal val) { + return compareTo(val) <= 0 ? this : val; + } + + public BigDecimal max(BigDecimal val) { + return compareTo(val) >= 0 ? this : val; + } + + /// Numeric order: `2.0` and `2.00` compare equal. + @Override + public int compareTo(BigDecimal val) { + if (signum != val.signum) { + return signum < val.signum ? -1 : 1; + } + if (signum == 0) { + return 0; + } + int s = Math.max(scale, val.scale); + int c = cmp(shift(mag, s - scale), shift(val.mag, s - val.scale)); + return signum > 0 ? c : -c; + } + + /// Equal in value AND scale: `2.0` does not equal `2.00`. + @Override + public boolean equals(Object x) { + if (x == this) { + return true; + } + if (!(x instanceof BigDecimal)) { + return false; + } + BigDecimal o = (BigDecimal) x; + return scale == o.scale && signum == o.signum && mag.equals(o.mag); + } + + /// The JDK's hash, `31 * unscaledValue().hashCode() + scale()`, where a + /// `BigInteger` hashes its big-endian 32-bit words. + @Override + public int hashCode() { + int h = 0; + if (signum != 0) { + // Little-endian 32-bit words of the magnitude. + int[] words = new int[mag.length() / 9 + 2]; + int n = 0; + String m = mag; + while (!"0".equals(m)) { + long[] rem = new long[1]; + m = divSmall(m, 1L << 32, rem); + words[n++] = (int) rem[0]; + } + for (int i = n - 1; i >= 0; i--) { + h = (int) (31 * h + (words[i] & 0xffffffffL)); + } + h *= signum; + } + return 31 * h + scale; + } + + /// The JDK's text: plain when the scale is not negative and the + /// adjusted exponent is at least -6, scientific otherwise. + @Override + public String toString() { + if (scale == 0) { + return signum < 0 ? "-" + mag : mag; + } + long adjusted = -(long) scale + (mag.length() - 1); + if (scale > 0 && adjusted >= -6) { + return toPlainString(); + } + StringBuilder sb = new StringBuilder(); + if (signum < 0) { + sb.append('-'); + } + sb.append(mag.charAt(0)); + if (mag.length() > 1) { + sb.append('.').append(mag, 1, mag.length()); + } + if (adjusted != 0) { + sb.append('E'); + if (adjusted > 0) { + sb.append('+'); + } + sb.append(adjusted); + } + return sb.toString(); + } + + /// The JDK's engineering notation is not provided; this is [#toString]. + public String toEngineeringString() { + return toString(); + } + + /// The value with no exponent. + public String toPlainString() { + StringBuilder sb = new StringBuilder(); + if (signum < 0) { + sb.append('-'); + } + if (scale <= 0) { + sb.append(mag); + if (signum != 0) { + for (int i = 0; i < -scale; i++) { + sb.append('0'); + } + } else if (scale < 0) { + // The JDK prints 0E+3 plainly as "0". + return "0"; + } + return sb.toString(); + } + int len = mag.length(); + if (len > scale) { + sb.append(mag, 0, len - scale).append('.').append(mag, len - scale, len); + } else { + sb.append("0."); + for (int i = len; i < scale; i++) { + sb.append('0'); + } + sb.append(mag); + } + return sb.toString(); + } + + /// The integer part, wrapped to 64 bits as the JDK's `longValue` does. + @Override + public long longValue() { + if (signum == 0) { + return 0; + } + int end = mag.length() - Math.max(scale, 0); + long v = 0; + for (int i = 0; i < end; i++) { + v = v * 10 + (mag.charAt(i) - '0'); + } + if (scale < 0) { + for (int i = 0; i < -scale && v != 0; i++) { + v *= 10; + } + } + return signum < 0 ? -v : v; + } + + /// The integer part; `ArithmeticException` if a fraction is discarded or + /// it does not fit in a long. + public long longValueExact() { + BigDecimal i = setScale(0, RoundingMode.UNNECESSARY); + if (i.mag.length() > 19 || (i.mag.length() == 19 + && cmp(i.mag, signum < 0 ? "9223372036854775808" : "9223372036854775807") > 0)) { + throw new ArithmeticException("Overflow"); + } + return i.longValue(); + } + + public int intValueExact() { + long v = longValueExact(); + if (v != (int) v) { + throw new ArithmeticException("Overflow"); + } + return (int) v; + } + + @Override + public int intValue() { + return (int) longValue(); + } + + @Override + public float floatValue() { + return (float) doubleValue(); + } + + @Override + public double doubleValue() { + return Double.parseDouble(toString()); + } + + // ---- decimal magnitude arithmetic ------------------------------------- + + private static int checkScale(long s) { + if (s > Integer.MAX_VALUE || s < Integer.MIN_VALUE) { + throw new ArithmeticException("Scale out of range"); + } + return (int) s; + } + + private static String magOf(long v) { + if (v == Long.MIN_VALUE) { + return "9223372036854775808"; + } + return Long.toString(v < 0 ? -v : v); + } + + private static String trim(String s) { + int i = 0; + while (i < s.length() - 1 && s.charAt(i) == '0') { + i++; + } + return i == 0 ? s : s.substring(i); + } + + /// `m * 10^n`. + private static String shift(String m, int n) { + if (n == 0 || "0".equals(m)) { + return m; + } + StringBuilder sb = new StringBuilder(m.length() + n).append(m); + for (int i = 0; i < n; i++) { + sb.append('0'); + } + return sb.toString(); + } + + private static int cmp(String a, String b) { + if (a.length() != b.length()) { + return a.length() < b.length() ? -1 : 1; + } + int c = a.compareTo(b); + return c < 0 ? -1 : (c > 0 ? 1 : 0); + } + + private static String add(String a, String b) { + StringBuilder sb = new StringBuilder(Math.max(a.length(), b.length()) + 1); + int i = a.length() - 1; + int j = b.length() - 1; + int carry = 0; + while (i >= 0 || j >= 0 || carry != 0) { + int d = carry + (i >= 0 ? a.charAt(i--) - '0' : 0) + (j >= 0 ? b.charAt(j--) - '0' : 0); + sb.append((char) ('0' + d % 10)); + carry = d / 10; + } + return trim(sb.reverse().toString()); + } + + /// `a - b` for `a >= b`. + private static String sub(String a, String b) { + StringBuilder sb = new StringBuilder(a.length()); + int i = a.length() - 1; + int j = b.length() - 1; + int borrow = 0; + while (i >= 0) { + int d = (a.charAt(i--) - '0') - borrow - (j >= 0 ? b.charAt(j--) - '0' : 0); + if (d < 0) { + d += 10; + borrow = 1; + } else { + borrow = 0; + } + sb.append((char) ('0' + d)); + } + return trim(sb.reverse().toString()); + } + + private static BigDecimal addSigned(int sa, String a, int sb, String b, int scale) { + if (sa == 0) { + return new BigDecimal(sb, b, scale); + } + if (sb == 0) { + return new BigDecimal(sa, a, scale); + } + if (sa == sb) { + return new BigDecimal(sa, add(a, b), scale); + } + int c = cmp(a, b); + if (c == 0) { + return new BigDecimal(0, "0", scale); + } + return c > 0 ? new BigDecimal(sa, sub(a, b), scale) : new BigDecimal(sb, sub(b, a), scale); + } + + private static String mul(String a, String b) { + if ("0".equals(a) || "0".equals(b)) { + return "0"; + } + int la = a.length(); + int lb = b.length(); + int[] r = new int[la + lb]; + for (int i = la - 1; i >= 0; i--) { + int da = a.charAt(i) - '0'; + if (da == 0) { + continue; + } + for (int j = lb - 1; j >= 0; j--) { + r[i + j + 1] += da * (b.charAt(j) - '0'); + } + // Normalise this row so no cell can overflow on large inputs. + for (int k = i + lb; k > i; k--) { + if (r[k] >= 10) { + r[k - 1] += r[k] / 10; + r[k] %= 10; + } + } + } + StringBuilder sb = new StringBuilder(r.length); + for (int k = 0; k < r.length; k++) { + sb.append((char) ('0' + r[k])); + } + return trim(sb.toString()); + } + + private static String pow(int base, int n) { + String r = "1"; + String b = Integer.toString(base); + while (n > 0) { + if ((n & 1) != 0) { + r = mul(r, b); + } + n >>= 1; + if (n > 0) { + b = mul(b, b); + } + } + return r; + } + + /// `m / d` for a small positive `d` (at most 2^32); the remainder goes + /// into `rem[0]`. + private static String divSmall(String m, long d, long[] rem) { + StringBuilder q = new StringBuilder(m.length()); + long r = 0; + for (int i = 0; i < m.length(); i++) { + r = r * 10 + (m.charAt(i) - '0'); + q.append((char) ('0' + (r / d))); + r %= d; + } + rem[0] = r; + return trim(q.toString()); + } + + /// `{a / b, a % b}` by long division, one decimal digit at a time. + private static String[] divmod(String a, String b) { + if (cmp(a, b) < 0) { + return new String[] {"0", a}; + } + StringBuilder q = new StringBuilder(a.length()); + String r = "0"; + for (int i = 0; i < a.length(); i++) { + r = "0".equals(r) ? String.valueOf(a.charAt(i)) : r + a.charAt(i); + r = trim(r); + int digit = 0; + while (cmp(r, b) >= 0) { + r = sub(r, b); + digit++; + } + q.append((char) ('0' + digit)); + } + return new String[] {trim(q.toString()), r}; + } + + /// The magnitude `q`, rounded by `mode` given the discarded remainder + /// `rem` of a division by `div`, for a result of sign `sign`. + private static String round(int sign, String q, String rem, String div, RoundingMode mode) { + if ("0".equals(rem)) { + return q; + } + boolean up; + switch (mode) { + case UP: + up = true; + break; + case DOWN: + up = false; + break; + case CEILING: + up = sign > 0; + break; + case FLOOR: + up = sign < 0; + break; + case UNNECESSARY: + throw new ArithmeticException("Rounding necessary"); + default: + int c = cmp(add(rem, rem), div); + if (mode == RoundingMode.HALF_UP) { + up = c >= 0; + } else if (mode == RoundingMode.HALF_DOWN) { + up = c > 0; + } else { + up = c > 0 || (c == 0 && ((q.charAt(q.length() - 1) - '0') & 1) != 0); + } + break; + } + return up ? add(q, "1") : q; + } +} diff --git a/vm/JavaAPI/src/java/math/RoundingMode.java b/vm/JavaAPI/src/java/math/RoundingMode.java new file mode 100644 index 00000000000..087ba423ecb --- /dev/null +++ b/vm/JavaAPI/src/java/math/RoundingMode.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.math; + +/// How a [BigDecimal] operation discards digits it cannot keep. +public enum RoundingMode { + UP, DOWN, CEILING, FLOOR, HALF_UP, HALF_DOWN, HALF_EVEN, UNNECESSARY; + + /// The mode for one of the legacy `BigDecimal.ROUND_*` constants, + /// which have the same order. + public static RoundingMode valueOf(int rm) { + RoundingMode[] all = values(); + if (rm < 0 || rm >= all.length) { + throw new IllegalArgumentException("argument out of range"); + } + return all[rm]; + } +} diff --git a/vm/JavaAPI/src/java/net/Inet4Address.java b/vm/JavaAPI/src/java/net/Inet4Address.java new file mode 100644 index 00000000000..31e288e83d2 --- /dev/null +++ b/vm/JavaAPI/src/java/net/Inet4Address.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.net; + +/// An IPv4 address, four bytes. +public final class Inet4Address extends InetAddress { + + Inet4Address(String hostName, byte[] address) { + super(hostName, address); + } + + @Override + public boolean isLoopbackAddress() { + return (addr[0] & 0xff) == 127; + } + + @Override + public boolean isAnyLocalAddress() { + return addr[0] == 0 && addr[1] == 0 && addr[2] == 0 && addr[3] == 0; + } + + @Override + public boolean isLinkLocalAddress() { + return (addr[0] & 0xff) == 169 && (addr[1] & 0xff) == 254; + } + + @Override + public boolean isSiteLocalAddress() { + int a = addr[0] & 0xff; + int b = addr[1] & 0xff; + return a == 10 || (a == 172 && (b & 0xf0) == 16) || (a == 192 && b == 168); + } + + @Override + public boolean isMulticastAddress() { + return (addr[0] & 0xf0) == 0xe0; + } + + @Override + public String getHostAddress() { + return (addr[0] & 0xff) + "." + (addr[1] & 0xff) + "." + (addr[2] & 0xff) + "." + (addr[3] & 0xff); + } + + @Override + public int hashCode() { + return ((addr[0] & 0xff) << 24) | ((addr[1] & 0xff) << 16) | ((addr[2] & 0xff) << 8) | (addr[3] & 0xff); + } +} diff --git a/vm/JavaAPI/src/java/net/Inet6Address.java b/vm/JavaAPI/src/java/net/Inet6Address.java new file mode 100644 index 00000000000..bba0d5e6e64 --- /dev/null +++ b/vm/JavaAPI/src/java/net/Inet6Address.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.net; + +/// An IPv6 address, sixteen bytes. Scope ids are not supported. +public final class Inet6Address extends InetAddress { + + Inet6Address(String hostName, byte[] address) { + super(hostName, address); + } + + private boolean allZeroTo(int end) { + for (int i = 0; i < end; i++) { + if (addr[i] != 0) { + return false; + } + } + return true; + } + + @Override + public boolean isLoopbackAddress() { + return allZeroTo(15) && addr[15] == 1; + } + + @Override + public boolean isAnyLocalAddress() { + return allZeroTo(16); + } + + @Override + public boolean isLinkLocalAddress() { + return (addr[0] & 0xff) == 0xfe && (addr[1] & 0xc0) == 0x80; + } + + @Override + public boolean isSiteLocalAddress() { + return (addr[0] & 0xff) == 0xfe && (addr[1] & 0xc0) == 0xc0; + } + + @Override + public boolean isMulticastAddress() { + return (addr[0] & 0xff) == 0xff; + } + + /// The JDK's form: eight groups of lowercase hex, not compressed. + @Override + public String getHostAddress() { + StringBuilder sb = new StringBuilder(39); + for (int i = 0; i < 16; i += 2) { + if (i > 0) { + sb.append(':'); + } + sb.append(Integer.toHexString(((addr[i] & 0xff) << 8) | (addr[i + 1] & 0xff))); + } + return sb.toString(); + } + + @Override + public int hashCode() { + int h = 0; + for (int i = 0; i < 16; i += 4) { + h += ((addr[i] & 0xff) << 24) | ((addr[i + 1] & 0xff) << 16) | ((addr[i + 2] & 0xff) << 8) + | (addr[i + 3] & 0xff); + } + return h; + } +} diff --git a/vm/JavaAPI/src/java/net/InetAddress.java b/vm/JavaAPI/src/java/net/InetAddress.java new file mode 100644 index 00000000000..d4a59adc7a7 --- /dev/null +++ b/vm/JavaAPI/src/java/net/InetAddress.java @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.net; + +/// An IP address, [Inet4Address] or [Inet6Address]. +/// +/// Only address LITERALS are resolved here: `getByName("10.0.0.1")`, +/// `getByName("::1")`, `getByName("[fe80::1]")`, plus `localhost` and the +/// empty or null name, which mean the loopback address as in the JDK. This +/// class library has no name resolver -- the backend's `getaddrinfo` lives +/// in its own TCP natives (`com.codename1.backend.Tcp`), which `vm/JavaAPI` +/// cannot depend on -- so any other host name throws +/// `UnknownHostException`, and [#getHostName] never does a reverse lookup. +public class InetAddress implements java.io.Serializable { + + final byte[] addr; + private final String hostName; + + InetAddress(String hostName, byte[] addr) { + this.hostName = hostName; + this.addr = addr; + } + + public static InetAddress getByName(String host) throws UnknownHostException { + return getAllByName(host)[0]; + } + + public static InetAddress[] getAllByName(String host) throws UnknownHostException { + if (host == null || host.length() == 0 || "localhost".equalsIgnoreCase(host)) { + return new InetAddress[] {getLoopbackAddress()}; + } + String literal = host; + if (literal.charAt(0) == '[') { + if (literal.length() < 3 || literal.charAt(literal.length() - 1) != ']') { + throw new UnknownHostException(host + ": invalid IPv6 address literal"); + } + literal = literal.substring(1, literal.length() - 1); + byte[] v6 = parseIpv6(literal); + if (v6 == null) { + throw new UnknownHostException(host + ": invalid IPv6 address literal"); + } + return new InetAddress[] {make(null, v6)}; + } + byte[] a = parseIpv4(literal); + if (a == null && literal.indexOf(':') >= 0) { + a = parseIpv6(literal); + if (a == null) { + throw new UnknownHostException(host + ": invalid IPv6 address literal"); + } + } + if (a == null) { + // No resolver in this class library; see the class comment. + throw new UnknownHostException(host); + } + return new InetAddress[] {make(null, a)}; + } + + public static InetAddress getByAddress(byte[] addr) throws UnknownHostException { + return getByAddress(null, addr); + } + + public static InetAddress getByAddress(String host, byte[] addr) throws UnknownHostException { + if (addr == null || (addr.length != 4 && addr.length != 16)) { + throw new UnknownHostException("addr is of illegal length"); + } + byte[] copy = new byte[addr.length]; + System.arraycopy(addr, 0, copy, 0, addr.length); + return make(host, copy); + } + + public static InetAddress getLoopbackAddress() { + return new Inet4Address("localhost", new byte[] {127, 0, 0, 1}); + } + + /// An IPv4-mapped IPv6 address becomes the IPv4 address, as in the JDK. + private static InetAddress make(String host, byte[] a) { + if (a.length == 16) { + boolean mapped = (a[10] & 0xff) == 0xff && (a[11] & 0xff) == 0xff; + for (int i = 0; i < 10 && mapped; i++) { + mapped = a[i] == 0; + } + if (mapped) { + return new Inet4Address(host, new byte[] {a[12], a[13], a[14], a[15]}); + } + return new Inet6Address(host, a); + } + return new Inet4Address(host, a); + } + + /// Four dot-separated decimal parts of 0..255, or null. + private static byte[] parseIpv4(String s) { + byte[] out = new byte[4]; + int part = 0; + int value = -1; + for (int i = 0; i <= s.length(); i++) { + char c = i < s.length() ? s.charAt(i) : '.'; + if (c == '.') { + if (value < 0 || part > 3) { + return null; + } + out[part++] = (byte) value; + value = -1; + } else if (c >= '0' && c <= '9') { + value = (value < 0 ? 0 : value * 10) + (c - '0'); + if (value > 255) { + return null; + } + } else { + return null; + } + } + return part == 4 ? out : null; + } + + /// RFC 4291 text form, with `::` and a trailing dotted IPv4 part, or null. + private static byte[] parseIpv6(String s) { + if (s.indexOf('%') >= 0) { + return null; + } + int dbl = s.indexOf("::"); + if (dbl >= 0 && s.indexOf("::", dbl + 1) >= 0) { + return null; + } + int[] head = new int[8]; + int[] tail = new int[8]; + int nh; + int nt = 0; + if (dbl >= 0) { + nh = groups(s.substring(0, dbl), head); + nt = groups(s.substring(dbl + 2), tail); + if (nh < 0 || nt < 0 || nh + nt > 7) { + return null; + } + } else { + nh = groups(s, head); + if (nh != 8) { + return null; + } + } + byte[] out = new byte[16]; + for (int i = 0; i < nh; i++) { + out[2 * i] = (byte) (head[i] >> 8); + out[2 * i + 1] = (byte) head[i]; + } + for (int i = 0; i < nt; i++) { + int g = 8 - nt + i; + out[2 * g] = (byte) (tail[i] >> 8); + out[2 * g + 1] = (byte) tail[i]; + } + return out; + } + + /// Parses colon-separated hex groups (the last may be dotted IPv4, worth + /// two groups) into `out`; the number of groups, or -1. + private static int groups(String s, int[] out) { + if (s.length() == 0) { + return 0; + } + int n = 0; + int start = 0; + while (true) { + int colon = s.indexOf(':', start); + String g = colon < 0 ? s.substring(start) : s.substring(start, colon); + if (colon < 0 && g.indexOf('.') >= 0) { + byte[] v4 = parseIpv4(g); + if (v4 == null || n + 2 > 8) { + return -1; + } + out[n++] = ((v4[0] & 0xff) << 8) | (v4[1] & 0xff); + out[n++] = ((v4[2] & 0xff) << 8) | (v4[3] & 0xff); + return n; + } + if (g.length() == 0 || g.length() > 4 || n >= 8) { + return -1; + } + int v = 0; + for (int i = 0; i < g.length(); i++) { + int d = Character.digit(g.charAt(i), 16); + if (d < 0) { + return -1; + } + v = v * 16 + d; + } + out[n++] = v; + if (colon < 0) { + return n; + } + start = colon + 1; + } + } + + /// A copy of the raw address, network byte order. + public byte[] getAddress() { + byte[] copy = new byte[addr.length]; + System.arraycopy(addr, 0, copy, 0, addr.length); + return copy; + } + + /// The address as text; overridden by both subclasses. + public String getHostAddress() { + return ""; + } + + /// The name this address was created with, or its text form; no + /// reverse lookup is done. + public String getHostName() { + return hostName != null ? hostName : getHostAddress(); + } + + public String getCanonicalHostName() { + return getHostName(); + } + + public boolean isLoopbackAddress() { + return false; + } + + public boolean isAnyLocalAddress() { + return false; + } + + public boolean isLinkLocalAddress() { + return false; + } + + public boolean isSiteLocalAddress() { + return false; + } + + public boolean isMulticastAddress() { + return false; + } + + /// Equal when the raw addresses are; the host name does not matter. + @Override + public boolean equals(Object obj) { + if (!(obj instanceof InetAddress)) { + return false; + } + byte[] o = ((InetAddress) obj).addr; + if (o.length != addr.length) { + return false; + } + for (int i = 0; i < o.length; i++) { + if (o[i] != addr[i]) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public String toString() { + return (hostName != null ? hostName : "") + "/" + getHostAddress(); + } +} diff --git a/vm/JavaAPI/src/java/net/UnknownHostException.java b/vm/JavaAPI/src/java/net/UnknownHostException.java new file mode 100644 index 00000000000..fe457c2efe9 --- /dev/null +++ b/vm/JavaAPI/src/java/net/UnknownHostException.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.net; + +/// A host name could not be resolved to an address. +public class UnknownHostException extends java.io.IOException { + + public UnknownHostException() { + } + + public UnknownHostException(String message) { + super(message); + } +} diff --git a/vm/JavaAPI/src/java/time/LocalDate.java b/vm/JavaAPI/src/java/time/LocalDate.java index 7c04bae2ff3..26c3af9a36f 100644 --- a/vm/JavaAPI/src/java/time/LocalDate.java +++ b/vm/JavaAPI/src/java/time/LocalDate.java @@ -27,6 +27,7 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.Temporal; import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalAdjuster; import java.time.temporal.TemporalAmount; import java.time.temporal.TemporalUnit; import java.util.function.LongFunction; @@ -34,7 +35,7 @@ import java.util.stream.Stream; /// A date of the ISO calendar with no time and no zone, such as 2007-12-03. -public final class LocalDate implements ChronoLocalDate { +public final class LocalDate implements ChronoLocalDate, TemporalAdjuster { /// -999999999-01-01, the earliest date. public static final LocalDate MIN = new LocalDate(-999999999, 1, 1); /// +999999999-12-31, the latest date. @@ -268,6 +269,28 @@ private long getProlepticMonth() { return (year * 12L + month - 1); } + /// This date moved by `adjuster`, for example + /// `TemporalAdjusters.lastDayOfMonth()`. + public LocalDate with(TemporalAdjuster adjuster) { + if (adjuster instanceof LocalDate) { + return (LocalDate) adjuster; + } + Temporal t = adjuster.adjustInto(this); + if (t instanceof LocalDate) { + return (LocalDate) t; + } + throw new DateTimeException("Adjuster did not return a LocalDate"); + } + + /// This date, as the adjustment of another `LocalDate`. + @Override + public Temporal adjustInto(Temporal temporal) { + if (temporal instanceof LocalDate) { + return this; + } + throw new DateTimeException("Cannot adjust " + temporal + " to a LocalDate"); + } + public LocalDate withYear(int year) { if (this.year == year) { return this; diff --git a/vm/JavaAPI/src/java/time/YearMonth.java b/vm/JavaAPI/src/java/time/YearMonth.java new file mode 100644 index 00000000000..3cfae6a1f13 --- /dev/null +++ b/vm/JavaAPI/src/java/time/YearMonth.java @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.time; + +/// A month of a year, such as `2024-02`, with no day and no zone. +public final class YearMonth implements Comparable, java.io.Serializable { + + private final int year; + private final int month; + + private YearMonth(int year, int month) { + this.year = year; + this.month = month; + } + + public static YearMonth of(int year, int month) { + if (month < 1 || month > 12) { + throw new DateTimeException("Invalid value for MonthOfYear: " + month); + } + if (year < -999999999 || year > 999999999) { + throw new DateTimeException("Invalid value for Year: " + year); + } + return new YearMonth(year, month); + } + + public static YearMonth of(int year, Month month) { + return of(year, month.getValue()); + } + + public static YearMonth now() { + return from(LocalDate.now()); + } + + public static YearMonth now(ZoneId zone) { + return from(LocalDate.now(zone)); + } + + public static YearMonth now(Clock clock) { + return from(LocalDate.now(clock)); + } + + /// The month of `date`. + public static YearMonth from(LocalDate date) { + return new YearMonth(date.getYear(), date.getMonthValue()); + } + + /// Parses `uuuu-MM`, with an optional sign on the year. + public static YearMonth parse(CharSequence text) { + String s = text.toString(); + int dash = s.lastIndexOf('-'); + if (dash <= 0 || s.length() - dash != 3) { + throw new java.time.format.DateTimeParseException("Text '" + s + "' could not be parsed", s, 0); + } + String y = s.substring(0, dash); + String m = s.substring(dash + 1); + int start = y.charAt(0) == '+' || y.charAt(0) == '-' ? 1 : 0; + // As the JDK's formatter: more than four year digits need a sign, + // and a '+' is only allowed on more than four. + int yearDigits = y.length() - start; + boolean plus = y.charAt(0) == '+'; + if (yearDigits < 4 || (yearDigits > 4 && start == 0) || (plus && yearDigits == 4) + || !digits(y, start) || !digits(m, 0)) { + throw new java.time.format.DateTimeParseException("Text '" + s + "' could not be parsed", s, 0); + } + if (y.charAt(0) == '+') { + y = y.substring(1); + } + try { + return of(Integer.parseInt(y), Integer.parseInt(m)); + } catch (NumberFormatException e) { + throw new java.time.format.DateTimeParseException("Text '" + s + "' could not be parsed", s, 0); + } catch (DateTimeException e) { + throw new java.time.format.DateTimeParseException("Text '" + s + "' could not be parsed: " + + e.getMessage(), s, 0); + } + } + + private static boolean digits(String s, int from) { + if (from >= s.length()) { + return false; + } + for (int i = from; i < s.length(); i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + return false; + } + } + return true; + } + + public int getYear() { + return year; + } + + public int getMonthValue() { + return month; + } + + public Month getMonth() { + return Month.of(month); + } + + public boolean isLeapYear() { + return (year & 3) == 0 && (year % 100 != 0 || year % 400 == 0); + } + + public int lengthOfMonth() { + return LocalDate.of(year, month, 1).lengthOfMonth(); + } + + public int lengthOfYear() { + return isLeapYear() ? 366 : 365; + } + + public boolean isValidDay(int dayOfMonth) { + return dayOfMonth >= 1 && dayOfMonth <= lengthOfMonth(); + } + + public YearMonth withYear(int year) { + return of(year, month); + } + + public YearMonth withMonth(int month) { + return of(year, month); + } + + public YearMonth plusMonths(long monthsToAdd) { + if (monthsToAdd == 0) { + return this; + } + long total = year * 12L + (month - 1) + monthsToAdd; + long y = Math.floorDiv(total, 12L); + if (y < -999999999L || y > 999999999L) { + throw new DateTimeException("Invalid value for Year: " + y); + } + return new YearMonth((int) y, (int) Math.floorMod(total, 12L) + 1); + } + + public YearMonth minusMonths(long monthsToSubtract) { + if (monthsToSubtract == Long.MIN_VALUE) { + return plusMonths(Long.MAX_VALUE).plusMonths(1); + } + return plusMonths(-monthsToSubtract); + } + + public YearMonth plusYears(long yearsToAdd) { + if (yearsToAdd == 0) { + return this; + } + long y = year + yearsToAdd; + if (y < -999999999L || y > 999999999L) { + throw new DateTimeException("Invalid value for Year: " + y); + } + return new YearMonth((int) y, month); + } + + public YearMonth minusYears(long yearsToSubtract) { + if (yearsToSubtract == Long.MIN_VALUE) { + return plusYears(Long.MAX_VALUE).plusYears(1); + } + return plusYears(-yearsToSubtract); + } + + public LocalDate atDay(int dayOfMonth) { + return LocalDate.of(year, month, dayOfMonth); + } + + public LocalDate atEndOfMonth() { + return LocalDate.of(year, month, lengthOfMonth()); + } + + public boolean isBefore(YearMonth other) { + return compareTo(other) < 0; + } + + public boolean isAfter(YearMonth other) { + return compareTo(other) > 0; + } + + @Override + public int compareTo(YearMonth other) { + int cmp = year - other.year; + if (cmp == 0) { + cmp = month - other.month; + } + return cmp; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof YearMonth)) { + return false; + } + YearMonth o = (YearMonth) obj; + return year == o.year && month == o.month; + } + + @Override + public int hashCode() { + return year ^ (month << 27); + } + + /// The JDK's text, `uuuu-MM`. Unlike `LocalDate`, the JDK puts no `+` + /// on a year past 9999 here, so such a text does not parse back. + @Override + public String toString() { + StringBuilder buf = new StringBuilder(10); + int absYear = year < 0 ? -year : year; + if (absYear < 1000) { + if (year < 0) { + buf.append('-'); + } + buf.append(Integer.toString(absYear + 10000).substring(1)); + } else { + buf.append(year); + } + buf.append(month < 10 ? "-0" : "-").append(month); + return buf.toString(); + } +} diff --git a/vm/JavaAPI/src/java/time/temporal/TemporalAdjuster.java b/vm/JavaAPI/src/java/time/temporal/TemporalAdjuster.java new file mode 100644 index 00000000000..a30be954e70 --- /dev/null +++ b/vm/JavaAPI/src/java/time/temporal/TemporalAdjuster.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.time.temporal; + +/// A strategy that moves a date to another one, such as "the last day of +/// the month". [TemporalAdjusters] has the common ones. +public interface TemporalAdjuster { + /// `temporal` adjusted; the result is of the same type. + Temporal adjustInto(Temporal temporal); +} diff --git a/vm/JavaAPI/src/java/time/temporal/TemporalAdjusters.java b/vm/JavaAPI/src/java/time/temporal/TemporalAdjusters.java new file mode 100644 index 00000000000..0cf7fbc151d --- /dev/null +++ b/vm/JavaAPI/src/java/time/temporal/TemporalAdjusters.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.time.temporal; + +import java.time.DayOfWeek; +import java.time.LocalDate; + +/// The common [TemporalAdjuster]s. They adjust a `LocalDate`; any other +/// temporal throws `UnsupportedTemporalTypeException`, because this class +/// library has no field-based `Temporal.with` for them to go through. +public final class TemporalAdjusters { + + private static final int FIRST_OF_MONTH = 0; + private static final int LAST_OF_MONTH = 1; + private static final int FIRST_OF_NEXT_MONTH = 2; + private static final int FIRST_OF_YEAR = 3; + private static final int LAST_OF_YEAR = 4; + private static final int FIRST_OF_NEXT_YEAR = 5; + + private TemporalAdjusters() { + } + + public static TemporalAdjuster firstDayOfMonth() { + return new Fixed(FIRST_OF_MONTH); + } + + public static TemporalAdjuster lastDayOfMonth() { + return new Fixed(LAST_OF_MONTH); + } + + public static TemporalAdjuster firstDayOfNextMonth() { + return new Fixed(FIRST_OF_NEXT_MONTH); + } + + public static TemporalAdjuster firstDayOfYear() { + return new Fixed(FIRST_OF_YEAR); + } + + public static TemporalAdjuster lastDayOfYear() { + return new Fixed(LAST_OF_YEAR); + } + + public static TemporalAdjuster firstDayOfNextYear() { + return new Fixed(FIRST_OF_NEXT_YEAR); + } + + /// The next `dayOfWeek` strictly after the date. + public static TemporalAdjuster next(DayOfWeek dayOfWeek) { + return new Relative(dayOfWeek, true, false); + } + + /// The next `dayOfWeek`, or the date itself when it already is one. + public static TemporalAdjuster nextOrSame(DayOfWeek dayOfWeek) { + return new Relative(dayOfWeek, true, true); + } + + /// The previous `dayOfWeek` strictly before the date. + public static TemporalAdjuster previous(DayOfWeek dayOfWeek) { + return new Relative(dayOfWeek, false, false); + } + + /// The previous `dayOfWeek`, or the date itself when it already is one. + public static TemporalAdjuster previousOrSame(DayOfWeek dayOfWeek) { + return new Relative(dayOfWeek, false, true); + } + + private static LocalDate date(Temporal temporal) { + if (temporal instanceof LocalDate) { + return (LocalDate) temporal; + } + throw new UnsupportedTemporalTypeException("Only LocalDate can be adjusted"); + } + + private static final class Fixed implements TemporalAdjuster { + private final int kind; + + Fixed(int kind) { + this.kind = kind; + } + + @Override + public Temporal adjustInto(Temporal temporal) { + LocalDate d = date(temporal); + switch (kind) { + case FIRST_OF_MONTH: + return d.withDayOfMonth(1); + case LAST_OF_MONTH: + return d.withDayOfMonth(d.lengthOfMonth()); + case FIRST_OF_NEXT_MONTH: + return d.withDayOfMonth(1).plusMonths(1); + case FIRST_OF_YEAR: + return d.withDayOfYear(1); + case LAST_OF_YEAR: + return d.withDayOfYear(d.lengthOfYear()); + default: + return d.withDayOfYear(1).plusYears(1); + } + } + } + + private static final class Relative implements TemporalAdjuster { + private final int dow; + private final boolean forward; + private final boolean orSame; + + Relative(DayOfWeek dayOfWeek, boolean forward, boolean orSame) { + if (dayOfWeek == null) { + throw new NullPointerException("dayOfWeek"); + } + this.dow = dayOfWeek.getValue(); + this.forward = forward; + this.orSame = orSame; + } + + @Override + public Temporal adjustInto(Temporal temporal) { + LocalDate d = date(temporal); + int cur = d.getDayOfWeek().getValue(); + if (orSame && cur == dow) { + return d; + } + if (forward) { + int diff = dow - cur; + return d.plusDays(diff <= 0 ? diff + 7 : diff); + } + int diff = cur - dow; + return d.minusDays(diff <= 0 ? diff + 7 : diff); + } + } +} diff --git a/vm/JavaAPI/src/java/util/concurrent/ConcurrentLinkedQueue.java b/vm/JavaAPI/src/java/util/concurrent/ConcurrentLinkedQueue.java new file mode 100644 index 00000000000..2c50626cc0f --- /dev/null +++ b/vm/JavaAPI/src/java/util/concurrent/ConcurrentLinkedQueue.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.util.concurrent; + +import java.util.AbstractQueue; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.NoSuchElementException; + +/// `java.util.concurrent.ConcurrentLinkedQueue`: an unbounded thread safe +/// FIFO queue. It is lock based rather than lock free, like +/// `LinkedBlockingQueue` here: every operation holds the list's monitor. +/// Iteration walks a snapshot taken when the iterator was created, so it +/// never throws `ConcurrentModificationException`; `remove` on the iterator +/// removes that element from the queue. +public class ConcurrentLinkedQueue extends AbstractQueue implements java.io.Serializable { + + private final LinkedList list = new LinkedList(); + + public ConcurrentLinkedQueue() { + } + + public ConcurrentLinkedQueue(Collection c) { + for (E e : c) { + offer(e); + } + } + + @Override + public boolean offer(E e) { + if (e == null) { + throw new NullPointerException(); + } + synchronized (list) { + list.addLast(e); + } + return true; + } + + @Override + public boolean add(E e) { + return offer(e); + } + + @Override + public E poll() { + synchronized (list) { + return list.isEmpty() ? null : list.removeFirst(); + } + } + + @Override + public E peek() { + synchronized (list) { + return list.isEmpty() ? null : list.getFirst(); + } + } + + @Override + public int size() { + synchronized (list) { + return list.size(); + } + } + + @Override + public boolean isEmpty() { + synchronized (list) { + return list.isEmpty(); + } + } + + @Override + public boolean contains(Object o) { + if (o == null) { + return false; + } + synchronized (list) { + return list.contains(o); + } + } + + @Override + public boolean remove(Object o) { + if (o == null) { + return false; + } + synchronized (list) { + return list.remove(o); + } + } + + @Override + public void clear() { + synchronized (list) { + list.clear(); + } + } + + @Override + public Object[] toArray() { + synchronized (list) { + return list.toArray(); + } + } + + @Override + public T[] toArray(T[] a) { + synchronized (list) { + return list.toArray(a); + } + } + + @Override + public Iterator iterator() { + final ArrayList snapshot; + synchronized (list) { + snapshot = new ArrayList(list); + } + return new Iterator() { + private int next; + private E last; + + @Override + public boolean hasNext() { + return next < snapshot.size(); + } + + @Override + public E next() { + if (next >= snapshot.size()) { + throw new NoSuchElementException(); + } + last = snapshot.get(next++); + return last; + } + + @Override + public void remove() { + if (last == null) { + throw new IllegalStateException(); + } + synchronized (list) { + // This exact element, not one equal to it. + Iterator it = list.iterator(); + while (it.hasNext()) { + if (it.next() == last) { + it.remove(); + break; + } + } + } + last = null; + } + }; + } +} diff --git a/vm/JavaAPI/src/java/util/concurrent/Flow.java b/vm/JavaAPI/src/java/util/concurrent/Flow.java new file mode 100644 index 00000000000..ec1702b2bfc --- /dev/null +++ b/vm/JavaAPI/src/java/util/concurrent/Flow.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.util.concurrent; + +/// `java.util.concurrent.Flow`: the reactive-streams interfaces. A +/// [Publisher] hands items to a [Subscriber] as fast as its [Subscription] +/// asks for them. +public final class Flow { + + private Flow() { + } + + /// Something that produces items for subscribers. + public interface Publisher { + void subscribe(Subscriber subscriber); + } + + /// Something that receives items, then an error or completion. + public interface Subscriber { + void onSubscribe(Subscription subscription); + + void onNext(T item); + + void onError(Throwable throwable); + + void onComplete(); + } + + /// The link between one publisher and one subscriber. + public interface Subscription { + void request(long n); + + void cancel(); + } + + /// A subscriber that is also a publisher. + public interface Processor extends Subscriber, Publisher { + } + + /// The JDK's default buffer size for publishers, 256. + public static int defaultBufferSize() { + return 256; + } +} diff --git a/vm/JavaAPI/src/java/util/concurrent/Semaphore.java b/vm/JavaAPI/src/java/util/concurrent/Semaphore.java new file mode 100644 index 00000000000..83ad689ef19 --- /dev/null +++ b/vm/JavaAPI/src/java/util/concurrent/Semaphore.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package java.util.concurrent; + +/// `java.util.concurrent.Semaphore`: a count of permits that threads take +/// and give back. A thread that asks for more permits than are free waits +/// through the package's `BlockingWait`, so the UI thread keeps dispatching +/// events while it waits. The fairness flag is accepted and reported, but +/// waiters are not served in arrival order. +public class Semaphore implements java.io.Serializable { + + private final Object lock = new Object(); + private final boolean fair; + /// Guarded by `lock`. + private int permits; + + public Semaphore(int permits) { + this(permits, false); + } + + public Semaphore(int permits, boolean fair) { + this.permits = permits; + this.fair = fair; + } + + public boolean isFair() { + return fair; + } + + public void acquire() throws InterruptedException { + acquire(1); + } + + public void acquire(int n) throws InterruptedException { + check(n); + if (Thread.interrupted()) { + throw new InterruptedException(); + } + waitFor(n, BlockingWait.FOREVER); + } + + public void acquireUninterruptibly() { + acquireUninterruptibly(1); + } + + public void acquireUninterruptibly(int n) { + check(n); + boolean interrupted = false; + while (true) { + try { + waitFor(n, BlockingWait.FOREVER); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + public boolean tryAcquire() { + return tryAcquire(1); + } + + public boolean tryAcquire(int n) { + check(n); + synchronized (lock) { + if (permits >= n) { + permits -= n; + return true; + } + return false; + } + } + + public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { + return tryAcquire(1, timeout, unit); + } + + public boolean tryAcquire(int n, long timeout, TimeUnit unit) throws InterruptedException { + check(n); + if (Thread.interrupted()) { + throw new InterruptedException(); + } + return waitFor(n, BlockingWait.deadline(BlockingWait.millis(timeout, unit))); + } + + public void release() { + release(1); + } + + public void release(int n) { + check(n); + synchronized (lock) { + permits += n; + lock.notifyAll(); + } + } + + public int availablePermits() { + synchronized (lock) { + return permits; + } + } + + public int drainPermits() { + synchronized (lock) { + int p = permits; + if (p > 0) { + permits = 0; + return p; + } + return 0; + } + } + + protected void reducePermits(int reduction) { + check(reduction); + synchronized (lock) { + permits -= reduction; + } + } + + @Override + public String toString() { + return super.toString() + "[Permits = " + availablePermits() + "]"; + } + + private static void check(int n) { + if (n < 0) { + throw new IllegalArgumentException(); + } + } + + /// Takes `n` permits, waiting until `deadline`; false when it passed. + private boolean waitFor(final int n, long deadline) throws InterruptedException { + while (true) { + synchronized (lock) { + if (permits >= n) { + permits -= n; + return true; + } + } + if (deadline != BlockingWait.FOREVER && deadline - BlockingWait.now() <= 0) { + return false; + } + // Another thread may take the permits between the wait ending + // and the lock being taken again, hence the loop. + new BlockingWait(lock, deadline) { + @Override + boolean ready() { + return permits >= n; + } + }.await(); + } + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java b/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java new file mode 100644 index 00000000000..41aa3605837 --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.DateTimeException; +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.YearMonth; +import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Flow; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +/// Classes issue #6153 found missing from the class library: Semaphore, +/// ConcurrentLinkedQueue, Flow, YearMonth, TemporalAdjusters, BigDecimal, +/// RoundingMode and InetAddress, each against the JDK's answers. +public final class JdkGapChecks { + private JdkGapChecks() { + } + + private static final String[] DECIMALS = {"0", "0.00", "1", "-1", "1.5", "-2.5", "2.5", "0.125", "123.4500", + "-0.001", "1E+3", "1.23E-10", "99999999999999999999.99", "-7", "3", "0.1", "1000"}; + + public static void run() { + Bulk.check("JdkGap.semaphoreCountsPermits", () -> { + Semaphore s = new Semaphore(2, true); + Bulk.p("fair", s.isFair()); + Bulk.p("a1", s.tryAcquire()); + Bulk.p("a2", s.tryAcquire()); + Bulk.p("a3", s.tryAcquire()); + Bulk.p("timed", s.tryAcquire(20, TimeUnit.MILLISECONDS)); + s.release(3); + Bulk.p("avail", s.availablePermits()); + s.acquire(2); + Bulk.p("afterAcquire", s.availablePermits()); + Bulk.p("timedOk", s.tryAcquire(1, 20, TimeUnit.MILLISECONDS)); + s.release(4); + Bulk.p("drained", s.drainPermits()); + Bulk.pThrown("negative", IllegalArgumentException.class, () -> s.release(-1)); + final Semaphore gate = new Semaphore(0); + Thread t = new Thread(() -> gate.release()); + t.start(); + gate.acquire(); + t.join(); + Bulk.p("crossThread", gate.availablePermits()); + }); + Bulk.check("JdkGap.concurrentLinkedQueueIsFifo", () -> { + ConcurrentLinkedQueue q = new ConcurrentLinkedQueue(Arrays.asList("a", "b")); + q.offer("c"); + q.add("d"); + Bulk.p("size", q.size()); + Bulk.p("peek", q.peek()); + Bulk.p("poll", q.poll()); + Bulk.p("contains", q.contains("c")); + Bulk.p("remove", q.remove("c")); + Iterator it = q.iterator(); + it.next(); + it.remove(); + q.offer("e"); + List rest = new ArrayList(); + for (String s : q) { + rest.add(s); + } + Bulk.p("rest", rest.toString()); + Bulk.pThrown("nullOffer", NullPointerException.class, () -> q.offer(null)); + q.clear(); + Bulk.p("emptyPoll", q.poll()); + Bulk.p("isEmpty", q.isEmpty()); + }); + Bulk.check("JdkGap.flowWiresASubscription", () -> { + Bulk.p("buffer", Flow.defaultBufferSize()); + final List seen = new ArrayList(); + Flow.Publisher pub = sub -> { + sub.onSubscribe(new Flow.Subscription() { + @Override + public void request(long n) { + seen.add("request" + n); + } + + @Override + public void cancel() { + seen.add("cancel"); + } + }); + sub.onNext("x"); + sub.onComplete(); + }; + pub.subscribe(new Flow.Subscriber() { + @Override + public void onSubscribe(Flow.Subscription s) { + s.request(1); + } + + @Override + public void onNext(String item) { + seen.add("next" + item); + } + + @Override + public void onError(Throwable t) { + seen.add("error"); + } + + @Override + public void onComplete() { + seen.add("complete"); + } + }); + Bulk.p("seen", seen.toString()); + }); + Bulk.check("JdkGap.yearMonthMatchesTheJdk", () -> { + int[][] yms = {{2024, 2}, {2023, 2}, {1900, 2}, {2000, 12}, {-5, 1}, {12345, 7}, {1, 1}}; + for (int[] ym : yms) { + YearMonth m = YearMonth.of(ym[0], ym[1]); + String k = m.toString(); + Bulk.p(k + ".len", m.lengthOfMonth()); + Bulk.p(k + ".leap", m.isLeapYear()); + Bulk.p(k + ".plus13", m.plusMonths(13).toString()); + Bulk.p(k + ".minus14", m.minusMonths(14).toString()); + Bulk.p(k + ".plusYears", m.plusYears(3).toString()); + Bulk.p(k + ".atDay1", m.atDay(1).toString()); + Bulk.p(k + ".end", m.atEndOfMonth().toString()); + Bulk.p(k + ".month", m.getMonth().name()); + final YearMonth fm = m; + Bulk.p(k + ".parse", safe(() -> YearMonth.parse(fm.toString()).equals(fm))); + int c = m.compareTo(YearMonth.of(2000, 1)); + Bulk.p(k + ".vs2000", (c < 0 ? -1 : (c > 0 ? 1 : 0)) + ":" + m.isAfter(YearMonth.of(2000, 1))); + } + Bulk.p("before", YearMonth.of(2024, 1).isBefore(YearMonth.of(2024, 2))); + Bulk.p("hash", YearMonth.of(2024, 1).hashCode() == YearMonth.of(2024, 1).hashCode()); + Bulk.p("parse", YearMonth.parse("2024-03").toString()); + Bulk.p("parseSigned", safe(() -> YearMonth.parse("+12345-07"))); + Bulk.p("parseNegative", safe(() -> YearMonth.parse("-0005-01"))); + Bulk.pThrown("badMonth", DateTimeException.class, () -> YearMonth.of(2024, 13)); + Bulk.pThrown("badParse", DateTimeException.class, () -> YearMonth.parse("2024-3")); + Bulk.pThrown("badDay", DateTimeException.class, () -> YearMonth.of(2023, 2).atDay(29)); + }); + Bulk.check("JdkGap.temporalAdjustersMatchTheJdk", () -> { + LocalDate[] dates = {LocalDate.of(2024, 2, 10), LocalDate.of(2024, 2, 29), LocalDate.of(2023, 12, 31), + LocalDate.of(2024, 1, 1), LocalDate.of(2026, 11, 1)}; + for (LocalDate d : dates) { + String k = d.toString(); + Bulk.p(k + ".first", d.with(TemporalAdjusters.firstDayOfMonth()).toString()); + Bulk.p(k + ".last", d.with(TemporalAdjusters.lastDayOfMonth()).toString()); + Bulk.p(k + ".nextMonth", d.with(TemporalAdjusters.firstDayOfNextMonth()).toString()); + Bulk.p(k + ".firstYear", d.with(TemporalAdjusters.firstDayOfYear()).toString()); + Bulk.p(k + ".lastYear", d.with(TemporalAdjusters.lastDayOfYear()).toString()); + Bulk.p(k + ".nextYear", d.with(TemporalAdjusters.firstDayOfNextYear()).toString()); + for (DayOfWeek w : DayOfWeek.values()) { + Bulk.p(k + "." + w + ".next", d.with(TemporalAdjusters.next(w)).toString()); + Bulk.p(k + "." + w + ".nextOrSame", d.with(TemporalAdjusters.nextOrSame(w)).toString()); + Bulk.p(k + "." + w + ".prev", d.with(TemporalAdjusters.previous(w)).toString()); + Bulk.p(k + "." + w + ".prevOrSame", d.with(TemporalAdjusters.previousOrSame(w)).toString()); + } + } + Bulk.p("withDate", LocalDate.of(2024, 1, 1).with(LocalDate.of(1999, 5, 6)).toString()); + }); + Bulk.check("JdkGap.bigDecimalArithmeticMatchesTheJdk", () -> { + for (String a : DECIMALS) { + final BigDecimal x = new BigDecimal(a); + Bulk.p(a + ".str", x.toString()); + Bulk.p(a + ".plain", x.toPlainString()); + Bulk.p(a + ".scale", x.scale()); + Bulk.p(a + ".signum", x.signum()); + Bulk.p(a + ".hash", x.hashCode()); + Bulk.p(a + ".strip", x.stripTrailingZeros().toString()); + Bulk.p(a + ".neg", x.negate().toString()); + Bulk.p(a + ".abs", x.abs().toString()); + Bulk.p(a + ".long", x.longValue()); + Bulk.p(a + ".int", x.intValue()); + Bulk.p(a + ".double", x.doubleValue()); + for (final RoundingMode m : RoundingMode.values()) { + Bulk.p(a + ".set1." + m.name(), safe(() -> x.setScale(1, m))); + Bulk.p(a + ".set0." + m.name(), safe(() -> x.setScale(0, m))); + } + for (String b : DECIMALS) { + final BigDecimal y = new BigDecimal(b); + String k = a + "," + b; + Bulk.p(k + ".add", x.add(y).toString()); + Bulk.p(k + ".sub", x.subtract(y).toString()); + Bulk.p(k + ".mul", x.multiply(y).toString()); + Bulk.p(k + ".cmp", x.compareTo(y)); + Bulk.p(k + ".eq", x.equals(y)); + Bulk.p(k + ".div", safe(() -> x.divide(y))); + Bulk.p(k + ".div4he", safe(() -> x.divide(y, 4, RoundingMode.HALF_EVEN))); + Bulk.p(k + ".div2up", safe(() -> x.divide(y, 2, RoundingMode.UP))); + Bulk.p(k + ".divFloor", safe(() -> x.divide(y, RoundingMode.FLOOR))); + } + } + Bulk.p("valueOfLong", BigDecimal.valueOf(Long.MIN_VALUE).toString()); + Bulk.p("valueOfScaled", BigDecimal.valueOf(12345, 3).toString()); + Bulk.p("valueOfDouble", BigDecimal.valueOf(0.1).toString()); + Bulk.p("valueOfDoubleBig", BigDecimal.valueOf(1e20).toString()); + Bulk.p("exactDouble", new BigDecimal(0.1).toString()); + Bulk.p("exactDoubleNeg", new BigDecimal(-2.5e-3).toString()); + Bulk.p("exactDoubleBig", new BigDecimal(1e20).toString()); + Bulk.p("constants", BigDecimal.ZERO + "," + BigDecimal.ONE + "," + BigDecimal.TEN); + Bulk.p("bigLong", new BigDecimal("123456789012345678901234567890").longValue()); + Bulk.p("pow", new BigDecimal("1.1").pow(5).toString()); + Bulk.p("remainder", new BigDecimal("10.5").remainder(new BigDecimal("3")).toString()); + Bulk.pThrown("badString", NumberFormatException.class, () -> new BigDecimal("1.2.3")); + Bulk.pThrown("nonTerminating", ArithmeticException.class, () -> BigDecimal.ONE.divide(new BigDecimal("3"))); + Bulk.p("roundingValueOf", RoundingMode.valueOf(BigDecimal.ROUND_HALF_EVEN).name()); + }); + Bulk.check("JdkGap.inetAddressLiteralsMatchTheJdk", () -> { + String[] hosts = {"127.0.0.1", "10.1.2.3", "172.16.0.1", "172.32.0.1", "192.168.1.1", "169.254.1.1", + "0.0.0.0", "8.8.8.8", "224.0.0.1", "::1", "::", "fe80::1", "fec0::1", "2001:db8::ff00:42:8329", + "[2001:db8::1]", "::ffff:1.2.3.4", "1:2:3:4:5:6:7:8", "ff02::1"}; + for (String h : hosts) { + InetAddress a = InetAddress.getByName(h); + Bulk.p(h + ".v4", a instanceof Inet4Address); + Bulk.p(h + ".v6", a instanceof Inet6Address); + Bulk.p(h + ".addr", a.getHostAddress()); + Bulk.p(h + ".str", a.toString()); + Bulk.p(h + ".bytes", Arrays.toString(a.getAddress())); + Bulk.p(h + ".loop", a.isLoopbackAddress()); + Bulk.p(h + ".site", a.isSiteLocalAddress()); + Bulk.p(h + ".any", a.isAnyLocalAddress()); + Bulk.p(h + ".link", a.isLinkLocalAddress()); + Bulk.p(h + ".multi", a.isMulticastAddress()); + Bulk.p(h + ".all", InetAddress.getAllByName(h).length); + InetAddress same = InetAddress.getByAddress(a.getAddress()); + Bulk.p(h + ".eq", a.equals(same) && a.hashCode() == same.hashCode()); + if (a instanceof Inet4Address) { + Bulk.p(h + ".hash", a.hashCode()); + } + } + InetAddress named = InetAddress.getByAddress("db", new byte[] {10, 0, 0, 9}); + Bulk.p("named", named.toString()); + Bulk.p("namedHost", named.getHostName()); + Bulk.pThrown("badLength", UnknownHostException.class, () -> InetAddress.getByAddress(new byte[3])); + Bulk.pThrown("badV6", UnknownHostException.class, () -> InetAddress.getByName("1::2::3")); + }); + Bulk.cn1Only("JdkGap.inetAddressDoesNotResolveNames", () -> { + Bulk.expectEquals("UnknownHostException", + Bulk.thrown(UnknownHostException.class, () -> InetAddress.getByName("example.com")), + "a host name is not resolved"); + Bulk.expect(InetAddress.getByName("localhost").isLoopbackAddress(), "localhost is the loopback"); + }); + } + + /// What `c` answers as text, or the simple name of what it threw. + private static String safe(Op c) { + try { + return String.valueOf(c.call()); + } catch (Exception e) { + return e.getClass().getSimpleName(); + } + } + + private interface Op { + Object call() throws Exception; + } +} From be764f6a5135ba68abf5dd915c6d041ea6dc28d9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:57:19 +0200 Subject: [PATCH 03/25] backend-package: relocate java.util.regex and String regex calls onto the shared engine checkBackend now moves java.util.regex onto com.codename1.util.regex (the java/util/regex rows of Relocation.JDK_SHIMS) and rewrites String's regex members to StringRegex (the String rows of the client's rewrite rules), and the regex package (all but REDebugCompiler) is @SharedWithBackend so it ships in codenameone-backend. Issue #6153 item 2. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 2 + .../CharacterArrayCharacterIterator.java | 1 + .../util/regex/CharacterIterator.java | 1 + .../com/codename1/util/regex/MatchResult.java | 1 + .../src/com/codename1/util/regex/Matcher.java | 1 + .../src/com/codename1/util/regex/Pattern.java | 1 + .../util/regex/PatternSyntaxException.java | 1 + .../src/com/codename1/util/regex/RE.java | 1 + .../com/codename1/util/regex/RECharacter.java | 1 + .../com/codename1/util/regex/RECompiler.java | 1 + .../com/codename1/util/regex/REProgram.java | 1 + .../util/regex/RESyntaxException.java | 1 + .../src/com/codename1/util/regex/REUtil.java | 1 + .../util/regex/ReaderCharacterIterator.java | 1 + .../util/regex/StreamCharacterIterator.java | 1 + .../util/regex/StringCharacterIterator.java | 1 + .../codename1/util/regex/StringReader.java | 1 + .../com/codename1/util/regex/StringRegex.java | 1 + .../codename1/util/regex/UnicodeTables.java | 1 + .../codename1/maven/BytecodeCompliance.java | 86 +++++++++++++++++-- .../maven/BytecodeComplianceTest.java | 62 +++++++++++++ 21 files changed, 163 insertions(+), 5 deletions(-) diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index 20235303682..dd741207f0c 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -47,6 +47,7 @@ on: - 'CodenameOne/src/com/codename1/realtime/**' - 'CodenameOne/src/com/codename1/qrcode/QrCode.java' - 'CodenameOne/src/com/codename1/impl/MdnsEngine.java' + - 'CodenameOne/src/com/codename1/util/regex/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' @@ -134,6 +135,7 @@ on: - 'CodenameOne/src/com/codename1/realtime/**' - 'CodenameOne/src/com/codename1/qrcode/QrCode.java' - 'CodenameOne/src/com/codename1/impl/MdnsEngine.java' + - 'CodenameOne/src/com/codename1/util/regex/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' diff --git a/CodenameOne/src/com/codename1/util/regex/CharacterArrayCharacterIterator.java b/CodenameOne/src/com/codename1/util/regex/CharacterArrayCharacterIterator.java index 9a83a7e2a4d..b8f5e73d79f 100644 --- a/CodenameOne/src/com/codename1/util/regex/CharacterArrayCharacterIterator.java +++ b/CodenameOne/src/com/codename1/util/regex/CharacterArrayCharacterIterator.java @@ -18,6 +18,7 @@ package com.codename1.util.regex; /// Encapsulates char[] as CharacterIterator +@com.codename1.impl.SharedWithBackend public final class CharacterArrayCharacterIterator implements CharacterIterator { /// encapsulated private final char[] src; diff --git a/CodenameOne/src/com/codename1/util/regex/CharacterIterator.java b/CodenameOne/src/com/codename1/util/regex/CharacterIterator.java index 1ea8e731248..95b97af453c 100644 --- a/CodenameOne/src/com/codename1/util/regex/CharacterIterator.java +++ b/CodenameOne/src/com/codename1/util/regex/CharacterIterator.java @@ -19,6 +19,7 @@ /// Encapsulates different types of character sources - String, InputStream, ... /// Defines a set of common methods +@com.codename1.impl.SharedWithBackend public interface CharacterIterator { /// #### Returns /// diff --git a/CodenameOne/src/com/codename1/util/regex/MatchResult.java b/CodenameOne/src/com/codename1/util/regex/MatchResult.java index b2d71de20db..53a9f165f64 100644 --- a/CodenameOne/src/com/codename1/util/regex/MatchResult.java +++ b/CodenameOne/src/com/codename1/util/regex/MatchResult.java @@ -23,6 +23,7 @@ package com.codename1.util.regex; /// What one match found: the same shape as `java.util.regex.MatchResult`. +@com.codename1.impl.SharedWithBackend public interface MatchResult { int start(); diff --git a/CodenameOne/src/com/codename1/util/regex/Matcher.java b/CodenameOne/src/com/codename1/util/regex/Matcher.java index 3d3ffc5d4d3..ffa7be7c6b2 100644 --- a/CodenameOne/src/com/codename1/util/regex/Matcher.java +++ b/CodenameOne/src/com/codename1/util/regex/Matcher.java @@ -36,6 +36,7 @@ /// `find`, groups by number and by name, regions, and the replacement /// methods with the JDK's `$1` and `${name}` syntax. `hitEnd`, /// `requireEnd` and the two bounds switches are not provided. +@com.codename1.impl.SharedWithBackend public final class Matcher implements MatchResult { private Pattern pattern; diff --git a/CodenameOne/src/com/codename1/util/regex/Pattern.java b/CodenameOne/src/com/codename1/util/regex/Pattern.java index c6ed97252d1..bf82230a7aa 100644 --- a/CodenameOne/src/com/codename1/util/regex/Pattern.java +++ b/CodenameOne/src/com/codename1/util/regex/Pattern.java @@ -88,6 +88,7 @@ /// likely to have. A group repeated many thousand times /// over a long text recurses once for each repetition, as the JDK's does, /// on a stack that is smaller on a device. +@com.codename1.impl.SharedWithBackend public final class Pattern { public static final int UNIX_LINES = 0x01; diff --git a/CodenameOne/src/com/codename1/util/regex/PatternSyntaxException.java b/CodenameOne/src/com/codename1/util/regex/PatternSyntaxException.java index 6319c57348d..86b1d06dbc6 100644 --- a/CodenameOne/src/com/codename1/util/regex/PatternSyntaxException.java +++ b/CodenameOne/src/com/codename1/util/regex/PatternSyntaxException.java @@ -25,6 +25,7 @@ /// The same shape as `java.util.regex.PatternSyntaxException`: a pattern that is not one, or /// that uses a construct this runtime does not have -- the description says /// which. +@com.codename1.impl.SharedWithBackend public class PatternSyntaxException extends IllegalArgumentException { private final String description; diff --git a/CodenameOne/src/com/codename1/util/regex/RE.java b/CodenameOne/src/com/codename1/util/regex/RE.java index ec9291c47cc..1365e2aed4e 100644 --- a/CodenameOne/src/com/codename1/util/regex/RE.java +++ b/CodenameOne/src/com/codename1/util/regex/RE.java @@ -251,6 +251,7 @@ /// #### See also /// /// - RECompiler +@com.codename1.impl.SharedWithBackend public class RE { /// Specifies normal, case-sensitive matching behaviour. diff --git a/CodenameOne/src/com/codename1/util/regex/RECharacter.java b/CodenameOne/src/com/codename1/util/regex/RECharacter.java index 69072e7b635..5104a73302b 100644 --- a/CodenameOne/src/com/codename1/util/regex/RECharacter.java +++ b/CodenameOne/src/com/codename1/util/regex/RECharacter.java @@ -16,6 +16,7 @@ */ package com.codename1.util.regex; +@com.codename1.impl.SharedWithBackend public abstract class RECharacter { public static final byte UNASSIGNED = 0; diff --git a/CodenameOne/src/com/codename1/util/regex/RECompiler.java b/CodenameOne/src/com/codename1/util/regex/RECompiler.java index 7d41244438c..19dc9c71c4f 100644 --- a/CodenameOne/src/com/codename1/util/regex/RECompiler.java +++ b/CodenameOne/src/com/codename1/util/regex/RECompiler.java @@ -28,6 +28,7 @@ /// #### See also /// /// - RE +@com.codename1.impl.SharedWithBackend public class RECompiler { // Node flags static final int NODE_NORMAL = 0; // No flags (nothing special) diff --git a/CodenameOne/src/com/codename1/util/regex/REProgram.java b/CodenameOne/src/com/codename1/util/regex/REProgram.java index d977a2178ca..0d3f7ffd560 100644 --- a/CodenameOne/src/com/codename1/util/regex/REProgram.java +++ b/CodenameOne/src/com/codename1/util/regex/REProgram.java @@ -25,6 +25,7 @@ /// - RE /// /// - RECompiler +@com.codename1.impl.SharedWithBackend public class REProgram { static final int OPT_HASBACKREFS = 1; static final int OPT_HASBOL = 2; diff --git a/CodenameOne/src/com/codename1/util/regex/RESyntaxException.java b/CodenameOne/src/com/codename1/util/regex/RESyntaxException.java index 055bbc60142..72ca6f9afc9 100644 --- a/CodenameOne/src/com/codename1/util/regex/RESyntaxException.java +++ b/CodenameOne/src/com/codename1/util/regex/RESyntaxException.java @@ -22,6 +22,7 @@ /// a regular expression during development. /// If you are making regular expresion programs dynamically then you can catch it /// if you wish. But should not be forced to. +@com.codename1.impl.SharedWithBackend public class RESyntaxException extends RuntimeException { /// Constructor. /// diff --git a/CodenameOne/src/com/codename1/util/regex/REUtil.java b/CodenameOne/src/com/codename1/util/regex/REUtil.java index 2b4c037298b..6296840941c 100644 --- a/CodenameOne/src/com/codename1/util/regex/REUtil.java +++ b/CodenameOne/src/com/codename1/util/regex/REUtil.java @@ -18,6 +18,7 @@ package com.codename1.util.regex; /// This is a class that contains utility helper methods for this package. +@com.codename1.impl.SharedWithBackend public abstract class REUtil { /// complex: private static final String complexPrefix = "complex:"; diff --git a/CodenameOne/src/com/codename1/util/regex/ReaderCharacterIterator.java b/CodenameOne/src/com/codename1/util/regex/ReaderCharacterIterator.java index 0d46a0f18ea..dc986d3a69d 100644 --- a/CodenameOne/src/com/codename1/util/regex/ReaderCharacterIterator.java +++ b/CodenameOne/src/com/codename1/util/regex/ReaderCharacterIterator.java @@ -21,6 +21,7 @@ import java.io.Reader; /// Encapsulates java.io.Reader as CharacterIterator +@com.codename1.impl.SharedWithBackend public final class ReaderCharacterIterator implements CharacterIterator { /// Underlying reader private final Reader reader; diff --git a/CodenameOne/src/com/codename1/util/regex/StreamCharacterIterator.java b/CodenameOne/src/com/codename1/util/regex/StreamCharacterIterator.java index d80f9cdf7ae..e44e51e30cc 100644 --- a/CodenameOne/src/com/codename1/util/regex/StreamCharacterIterator.java +++ b/CodenameOne/src/com/codename1/util/regex/StreamCharacterIterator.java @@ -21,6 +21,7 @@ import java.io.InputStream; /// Encapsulates java.io.InputStream as CharacterIterator. +@com.codename1.impl.SharedWithBackend public final class StreamCharacterIterator implements CharacterIterator { /// Underlying is private final InputStream is; diff --git a/CodenameOne/src/com/codename1/util/regex/StringCharacterIterator.java b/CodenameOne/src/com/codename1/util/regex/StringCharacterIterator.java index d285df1bb30..e2bdb685fa5 100644 --- a/CodenameOne/src/com/codename1/util/regex/StringCharacterIterator.java +++ b/CodenameOne/src/com/codename1/util/regex/StringCharacterIterator.java @@ -18,6 +18,7 @@ package com.codename1.util.regex; /// Encapsulates String as CharacterIterator. +@com.codename1.impl.SharedWithBackend public final class StringCharacterIterator implements CharacterIterator { /// encapsulated private final String src; diff --git a/CodenameOne/src/com/codename1/util/regex/StringReader.java b/CodenameOne/src/com/codename1/util/regex/StringReader.java index ef45272d8de..f72634c21bc 100644 --- a/CodenameOne/src/com/codename1/util/regex/StringReader.java +++ b/CodenameOne/src/com/codename1/util/regex/StringReader.java @@ -18,6 +18,7 @@ import java.io.Reader; +@com.codename1.impl.SharedWithBackend public class StringReader extends Reader { private static final char NEW_LINE = '\n'; diff --git a/CodenameOne/src/com/codename1/util/regex/StringRegex.java b/CodenameOne/src/com/codename1/util/regex/StringRegex.java index 12f49e38cd9..5fdf2603bd8 100644 --- a/CodenameOne/src/com/codename1/util/regex/StringRegex.java +++ b/CodenameOne/src/com/codename1/util/regex/StringRegex.java @@ -30,6 +30,7 @@ /// this package's [Pattern]: `matches`, `split`, `replaceAll` and /// `replaceFirst`, with the JDK's syntax. Each takes the string as its /// first argument, so `s.split(";")` becomes `StringRegex.split(s, ";")`. +@com.codename1.impl.SharedWithBackend public final class StringRegex { /// The pattern compiled last. Code that splits every line of a file by diff --git a/CodenameOne/src/com/codename1/util/regex/UnicodeTables.java b/CodenameOne/src/com/codename1/util/regex/UnicodeTables.java index 3b03b2accea..98e028c3822 100644 --- a/CodenameOne/src/com/codename1/util/regex/UnicodeTables.java +++ b/CodenameOne/src/com/codename1/util/regex/UnicodeTables.java @@ -38,6 +38,7 @@ /// `UnicodeBlock.of` over every code point, with the script aliases and block /// names the JDK's `forName` methods accept (a block name's spaceless /// spelling is derived rather than stored). +@com.codename1.impl.SharedWithBackend final class UnicodeTables { private static final String DIGITS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/maven/build-engine/src/main/java/com/codename1/maven/BytecodeCompliance.java b/maven/build-engine/src/main/java/com/codename1/maven/BytecodeCompliance.java index a599178da95..f2bde1b1082 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/BytecodeCompliance.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/BytecodeCompliance.java @@ -38,6 +38,8 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.SimpleRemapper; import org.objectweb.asm.util.CheckClassAdapter; import org.objectweb.asm.tree.AbstractInsnNode; import org.objectweb.asm.tree.ClassNode; @@ -294,11 +296,83 @@ protected static long lastModifiedRecursive(File file, FilenameFilter filter) { }; private static final String SIMD_INTERNAL_NAME = "com/codename1/util/Simd"; private static final Map INVOCATION_REWRITE_RULES = createInvocationRewriteRules(); + /// The rules a backend gets: the String regex members alone. The other + /// rules point at the client's JdkApiRewriteHelper, which the server + /// runtime does not have; vm/JavaAPI defines those methods itself. + private static final Map BACKEND_INVOCATION_REWRITE_RULES = createBackendInvocationRewriteRules(); private File complianceOutputFile; private InvocationRewriteSummary lastInvocationRewriteSummary = new InvocationRewriteSummary(); private URLClassLoader validationClassLoader; + private static Map createBackendInvocationRewriteRules() { + Map rules = new LinkedHashMap(); + for (Map.Entry rule : INVOCATION_REWRITE_RULES.entrySet()) { + if (STRING_REGEX_INTERNAL_NAME.equals(rule.getValue().owner)) { + rules.put(rule.getKey(), rule.getValue()); + } + } + return Collections.unmodifiableMap(rules); + } + + /// The `java.util.regex` rows of [Relocation#JDK_SHIMS], the only rows a + /// backend applies: the server shares the framework's regex engine + /// (`com.codename1.util.regex`, marked `SharedWithBackend`), and nothing + /// else of that table. + private static Map backendRegexRelocation() { + Map map = new HashMap(); + for (String[] row : Relocation.JDK_SHIMS) { + if (row[0].startsWith("java/util/regex/")) { + map.put(row[0], row[1]); + } + } + return map; + } + + /// Moves a backend application's `java.util.regex` onto the framework's + /// engine and its String regex calls onto `StringRegex`, as a client build + /// does, IN PLACE. vm/JavaAPI has no `java.util.regex` and no String regex + /// members, so without this every such call is a forbidden reference. + private InvocationRewriteSummary applyBackendRegexRewrites(File output) throws BuildExecutionException { + InvocationRewriteSummary summary = new InvocationRewriteSummary(); + List classFiles = new ArrayList(); + collectClassFiles(output, classFiles); + final Map relocation = backendRegexRelocation(); + for (File classFile : classFiles) { + try { + byte[] original = FileUtils.readFileToByteArray(classFile); + final boolean[] moved = new boolean[1]; + ClassReader reader = new ClassReader(original); + ClassWriter writer = new ClassWriter(0); + reader.accept(new ClassRemapper(writer, new SimpleRemapper(relocation) { + @Override + public String map(String key) { + String mapped = super.map(key); + if (mapped != null) { + moved[0] = true; + } + return mapped; + } + }), 0); + byte[] relocated = moved[0] ? writer.toByteArray() : original; + InvocationRewriteResult result = rewriteClassInvocations(relocated, BACKEND_INVOCATION_REWRITE_RULES); + if (moved[0] || result.rewrittenCallsites > 0) { + FileUtils.writeByteArrayToFile(classFile, result.bytes); + summary.rewrittenClasses++; + summary.rewrittenCallsites += result.rewrittenCallsites; + } + } catch (IOException ex) { + throw new BuildExecutionException("Failed to rewrite regular expressions in " + classFile, ex); + } + } + return summary; + } + + /// Callsites the last check rewrote, as its report states them. + int lastRewrittenCallsites() { + return lastInvocationRewriteSummary.rewrittenCallsites; + } + private static Map createInvocationRewriteRules() { Map rules = new LinkedHashMap(); // ==================================================================== @@ -565,6 +639,7 @@ private File writePortReport(File outputDir, ComplianceFindings findings, List dependencies) throws BuildExecutionException { checkBackendBootstraps(output); + lastInvocationRewriteSummary = applyBackendRegexRewrites(output); complianceOutputFile = new File(output.getParentFile(), output.getName() + "-compliance.txt"); Map allowed = buildClassIndex(Collections.singletonList(javaApi)); Map project = buildClassIndexWithOutput(output, dependencies); @@ -907,7 +982,7 @@ private InvocationRewriteSummary applyInvocationRewrites(File outputDir) throws for (File classFile : classFiles) { try { byte[] originalBytes = FileUtils.readFileToByteArray(classFile); - InvocationRewriteResult rewriteResult = rewriteClassInvocations(originalBytes); + InvocationRewriteResult rewriteResult = rewriteClassInvocations(originalBytes, INVOCATION_REWRITE_RULES); if (rewriteResult.rewrittenCallsites > 0) { validateClass(rewriteResult.bytes, classFile, outputDir); FileUtils.writeByteArrayToFile(classFile, rewriteResult.bytes); @@ -928,10 +1003,11 @@ private InvocationRewriteSummary applyInvocationRewrites(File outputDir) throws /// `classBytes` with every invocation rewrite rule applied, as a build /// applies them: for tests that run a class the way it ships. static byte[] rewriteInvocations(byte[] classBytes) { - return rewriteClassInvocations(classBytes).bytes; + return rewriteClassInvocations(classBytes, INVOCATION_REWRITE_RULES).bytes; } - private static InvocationRewriteResult rewriteClassInvocations(byte[] classBytes) { + private static InvocationRewriteResult rewriteClassInvocations(byte[] classBytes, + final Map rules) { final InvocationRewriteResult result = new InvocationRewriteResult(); final ClassReader reader = new ClassReader(classBytes); final ClassWriter writer = new ClassWriter(reader, 0); @@ -943,7 +1019,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str @Override public void visitMethodInsn(int opcode, String owner, String methodName, String methodDescriptor, boolean isInterface) { MethodRef source = new MethodRef(opcode, owner, methodName, methodDescriptor); - MethodRef target = INVOCATION_REWRITE_RULES.get(source); + MethodRef target = rules.get(source); if (target != null) { result.rewrittenCallsites++; super.visitMethodInsn(target.opcode, target.owner, target.name, target.descriptor, false); @@ -964,7 +1040,7 @@ public void visitInvokeDynamicInsn(String indyName, String indyDescriptor, Handl int opcode = h.getTag() == Opcodes.H_INVOKESTATIC ? Opcodes.INVOKESTATIC : h.getTag() == Opcodes.H_INVOKEVIRTUAL ? Opcodes.INVOKEVIRTUAL : -1; MethodRef target = opcode < 0 ? null - : INVOCATION_REWRITE_RULES.get(new MethodRef(opcode, h.getOwner(), h.getName(), h.getDesc())); + : rules.get(new MethodRef(opcode, h.getOwner(), h.getName(), h.getDesc())); if (target == null) { continue; } diff --git a/maven/build-engine/src/test/java/com/codename1/maven/BytecodeComplianceTest.java b/maven/build-engine/src/test/java/com/codename1/maven/BytecodeComplianceTest.java index 62d2fa01071..d128dda0405 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/BytecodeComplianceTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/BytecodeComplianceTest.java @@ -88,6 +88,68 @@ void backendAllowsThirdPartyJavaxButProtectsNativeApi(@TempDir Path dir) throws compliance.checkBackend(output.toFile(), api.toFile(), Collections.singletonList(dependency.toFile()))); } + @Test + void backendMovesRegularExpressionsOntoTheSharedEngine(@TempDir Path dir) throws Exception { + Path src = Files.createDirectories(dir.resolve("src/app")); + Path output = Files.createDirectories(dir.resolve("output")); + Path api = Files.createDirectories(dir.resolve("api")); + Path dependency = Files.createDirectories(dir.resolve("dependency")); + Files.write(src.resolve("Regex.java"), java.util.Arrays.asList( + "package app;", + "public class Regex {", + " public static boolean run(String s) {", + " boolean a = java.util.regex.Pattern.compile(\"[a-z]+\").matcher(s).matches();", + " String[] p = s.split(\",\");", + " String r = s.replaceAll(\"x\", \"y\").replaceFirst(\"y\", \"z\");", + " return a && s.matches(\"b\") && p.length > 0 && r.length() > 0;", + " }", + "}"), java.nio.charset.StandardCharsets.UTF_8); + javax.tools.JavaCompiler javac = javax.tools.ToolProvider.getSystemJavaCompiler(); + assertEquals(0, javac.run(null, null, null, "-source", "8", "-target", "8", "-nowarn", + "-d", output.toString(), src.resolve("Regex.java").toString())); + + // The server's class library: no java.util.regex, no String regex members. + writeJavaLangObject(api); + writeStub(api, "java/lang/String", "length()I"); + writeStub(api, "java/lang/CharSequence"); + String r = "com/codename1/util/regex/"; + writeStub(dependency, r + "Pattern", "static compile(Ljava/lang/String;)L" + r + "Pattern;", + "matcher(Ljava/lang/CharSequence;)L" + r + "Matcher;"); + writeStub(dependency, r + "Matcher", "matches()Z"); + String str = "Ljava/lang/String;"; + writeStub(dependency, r + "StringRegex", "static matches(" + str + str + ")Z", + "static split(" + str + str + ")[" + str, + "static replaceAll(" + str + str + str + ")" + str, + "static replaceFirst(" + str + str + str + ")" + str); + + BytecodeCompliance compliance = new BytecodeCompliance(TestProjectHost.empty()); + compliance.checkBackend(output.toFile(), api.toFile(), Collections.singletonList(dependency.toFile())); + assertEquals(4, compliance.lastRewrittenCallsites()); + String rewritten = new String(Files.readAllBytes(output.resolve("app/Regex.class")), + java.nio.charset.StandardCharsets.ISO_8859_1); + org.junit.jupiter.api.Assertions.assertFalse(rewritten.contains("java/util/regex")); + org.junit.jupiter.api.Assertions.assertTrue(rewritten.contains(r + "StringRegex")); + } + + /// A class whose methods are native: present for the compliance index, + /// nothing to run. A method spec is `[static ]name descriptor`. + private void writeStub(Path root, String className, String... methods) throws Exception { + ClassWriter writer = new ClassWriter(0); + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, className, null, "java/lang/Object", null); + for (String m : methods) { + int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_NATIVE; + String spec = m; + if (spec.startsWith("static ")) { + access |= Opcodes.ACC_STATIC; + spec = spec.substring(7); + } + int paren = spec.indexOf('('); + writer.visitMethod(access, spec.substring(0, paren), spec.substring(paren), null, null).visitEnd(); + } + writer.visitEnd(); + writeBytes(root, className, writer.toByteArray()); + } + @Test void backendComplianceUsesTheFirstClasspathDefinition(@TempDir Path dir) throws Exception { Path output = Files.createDirectories(dir.resolve("output")); From 4ea28757fd236388cbf8d87ed3502d6bdbf265bc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:02:44 +0200 Subject: [PATCH 04/25] Backend: a logger facade with per-logger levels from configuration Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/backend/log/LoggerTest.java | 123 +++++++++++ .../src/com/codename1/backend/Backend.java | 3 + .../src/com/codename1/backend/log/Level.java | 53 +++++ .../com/codename1/backend/log/LogSink.java | 36 ++++ .../src/com/codename1/backend/log/Logger.java | 195 ++++++++++++++++++ .../com/codename1/backend/log/Logging.java | 194 +++++++++++++++++ .../src/com/codename1/backend/log/MDC.java | 109 ++++++++++ .../com/codename1/backend/log/Timestamps.java | 60 ++++++ .../codename1/backend/log/package-info.java | 26 +++ 9 files changed, 799 insertions(+) create mode 100644 maven/backend/src/test/java/com/codename1/backend/log/LoggerTest.java create mode 100644 vm/backend/src/com/codename1/backend/log/Level.java create mode 100644 vm/backend/src/com/codename1/backend/log/LogSink.java create mode 100644 vm/backend/src/com/codename1/backend/log/Logger.java create mode 100644 vm/backend/src/com/codename1/backend/log/Logging.java create mode 100644 vm/backend/src/com/codename1/backend/log/MDC.java create mode 100644 vm/backend/src/com/codename1/backend/log/Timestamps.java create mode 100644 vm/backend/src/com/codename1/backend/log/package-info.java diff --git a/maven/backend/src/test/java/com/codename1/backend/log/LoggerTest.java b/maven/backend/src/test/java/com/codename1/backend/log/LoggerTest.java new file mode 100644 index 00000000000..a788bc56d36 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/log/LoggerTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +import com.codename1.backend.Config; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LoggerTest { + private final List lines = new ArrayList(); + private final List errors = new ArrayList(); + + private void capture() { + Logging.setSink(new LogSink() { + @Override + public void write(Level level, String logger, String line, Throwable error) { + lines.add(line); + errors.add(error); + } + }); + } + + @AfterEach + void reset() { + Logging.setSink(null); + Logging.setLevel("root", null); + Logging.setLevel("com.example", null); + Logging.setLevel("com.example.push", null); + MDC.clear(); + } + + @Test + void placeholdersAndAThrowableLast() { + capture(); + Logger log = Logger.getLogger("com.example.Push"); + IllegalStateException failure = new IllegalStateException("x"); + log.info("sent {} to {} \\{}", 3, "app", failure); + assertEquals(1, lines.size()); + String line = lines.get(0); + assertTrue(line.matches("\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d\\.\\d{3}Z INFO \\[.+\\] com\\.example\\.Push - sent 3 to app \\{\\}"), line); + assertSame(failure, errors.get(0)); + log.warn("only {}", failure); + assertNull(errors.get(1), "a throwable a placeholder took is not also the stack trace"); + } + + @Test + void levelsComeFromTheLongestConfiguredName() throws Exception { + Properties p = new Properties(); + p.setProperty("logging.level.root", "warn"); + p.setProperty("logging.level.com.example", "INFO"); + p.setProperty("logging.level.com.example.push", "debug"); + Logging.configure(Config.of(p, null)); + assertTrue(Logger.getLogger("com.example.push.Apns").isDebugEnabled()); + assertFalse(Logger.getLogger("com.example.pushy.X").isDebugEnabled()); + assertTrue(Logger.getLogger("com.example.pushy.X").isInfoEnabled()); + assertFalse(Logger.getLogger("org.other.Y").isInfoEnabled()); + assertTrue(Logger.getLogger("org.other.Y").isWarnEnabled()); + Logger cached = Logger.getLogger("org.other.Y"); + assertFalse(cached.isInfoEnabled()); + Logging.setLevel("org.other", Level.TRACE); + assertTrue(cached.isTraceEnabled(), "a change reaches a logger that cached its level"); + } + + @Test + void anUnknownLevelIsRefused() { + Properties p = new Properties(); + p.setProperty("logging.level.com.example", "LOUD"); + assertThrows(java.io.IOException.class, () -> Logging.configure(Config.of(p, null))); + } + + @Test + void theContextMapIsAppended() { + capture(); + MDC.put("request", "r-1"); + Logger.getLogger("a").info("hi"); + MDC.remove("request"); + Logger.getLogger("a").info("bye"); + assertTrue(lines.get(0).endsWith(" - hi {request=r-1}"), lines.get(0)); + assertTrue(lines.get(1).endsWith(" - bye"), lines.get(1)); + } + + @Test + void isoTimestamps() { + StringBuilder out = new StringBuilder(); + Timestamps.appendIso(out, 0L); + out.append(' '); + Timestamps.appendIso(out, 951782400123L); + out.append(' '); + Timestamps.appendIso(out, -1L); + assertEquals("1970-01-01T00:00:00.000Z 2000-02-29T00:00:00.123Z 1969-12-31T23:59:59.999Z", out.toString()); + } +} diff --git a/vm/backend/src/com/codename1/backend/Backend.java b/vm/backend/src/com/codename1/backend/Backend.java index e4fdd2c27f9..d6e07ed79e4 100644 --- a/vm/backend/src/com/codename1/backend/Backend.java +++ b/vm/backend/src/com/codename1/backend/Backend.java @@ -1837,6 +1837,9 @@ private Backend startOnce() throws Exception { config = Config.load(); } config = config.withCompiledDefaults(compiledSettings); + // Before anything logs: the logging.level.* settings decide what the + // rest of the start-up writes. + com.codename1.backend.log.Logging.configure(config); // Before anything is opened: a setting only the build could have // acted on is refused here rather than silently doing nothing. BuildTimeSettings.require(config, managementRoute != null || testApplication, diff --git a/vm/backend/src/com/codename1/backend/log/Level.java b/vm/backend/src/com/codename1/backend/log/Level.java new file mode 100644 index 00000000000..419968338d1 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/Level.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +/// The severity of a log line, lowest first. A logger set to a level writes +/// that level and every one above it. +public enum Level { + TRACE, DEBUG, INFO, WARN, ERROR, OFF; + + /// The level a configuration value names, ignoring case, or null for one this + /// does not know. Compared without `toUpperCase`, which is locale sensitive: + /// on a Turkish host "info" would not fold to "INFO". + public static Level parse(String value) { + if (value == null) { + return null; + } + String v = value.trim(); + Level[] all = values(); + for (int i = 0; i < all.length; i++) { + if (all[i].name().equalsIgnoreCase(v)) { + return all[i]; + } + } + // Spring Boot also accepts these two names for the extremes. + if ("FATAL".equalsIgnoreCase(v)) { + return ERROR; + } + if ("ALL".equalsIgnoreCase(v)) { + return TRACE; + } + return null; + } +} diff --git a/vm/backend/src/com/codename1/backend/log/LogSink.java b/vm/backend/src/com/codename1/backend/log/LogSink.java new file mode 100644 index 00000000000..a5cd804a5f1 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/LogSink.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +/// Where log lines go. The default writes each line to standard output, the +/// stream a container platform collects; install another with +/// [Logging#setSink(LogSink)] to forward them elsewhere. +public interface LogSink { + /// Writes one line. + /// + /// @param level the line's level + /// @param logger the name of the logger that wrote it + /// @param line the whole formatted line, timestamp to message, with no newline + /// @param error the throwable passed with the message, or null + void write(Level level, String logger, String line, Throwable error); +} diff --git a/vm/backend/src/com/codename1/backend/log/Logger.java b/vm/backend/src/com/codename1/backend/log/Logger.java new file mode 100644 index 00000000000..b5a85a2d038 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/Logger.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +/// An application logger in the shape slf4j's is, writing through [Logging]. +/// +/// ```java +/// private static final Logger LOG = Logger.getLogger(PushService.class); +/// +/// LOG.info("sent {} notifications to {}", count, appId); +/// LOG.warn("APNs rejected token", ex); +/// if (LOG.isDebugEnabled()) { +/// LOG.debug("payload {}", render(payload)); +/// } +/// ``` +/// +/// Each `{}` in the message is replaced by the next argument; `\{}` is a +/// literal `{}`. A throwable left over after the placeholders are filled -- the +/// last argument, as slf4j takes it -- is written with its stack trace. +/// +/// A line reads `2026-11-01T09:30:00.123Z INFO [thread] com.example.Push - message`, +/// followed by the thread's [MDC] context when it has one. The level of each +/// logger comes from the `logging.level.*` settings; see [Logging]. +/// +/// No reflection, no configuration file and no static binding: a logger is a +/// name and a cached level. +public final class Logger { + private final String name; + /// The level last resolved and the configuration generation it came from. + /// Read and written without synchronisation on purpose: a racing reader at + /// worst resolves the level again, which gives the same answer. + private Level level; + private int generation = -1; + + private Logger(String name) { + this.name = name; + } + + /// The logger named after `type`, its fully qualified class name. + public static Logger getLogger(Class type) { + return new Logger(type.getName()); + } + + /// The logger with `name`. + public static Logger getLogger(String name) { + if (name == null) { + throw new IllegalArgumentException("name is null"); + } + return new Logger(name); + } + + /// The name lines are written under. + public String getName() { + return name; + } + + /// Whether a line at `level` would be written. + public boolean isEnabled(Level level) { + return level != Level.OFF && level.compareTo(effectiveLevel()) >= 0; + } + + public boolean isTraceEnabled() { + return isEnabled(Level.TRACE); + } + + public boolean isDebugEnabled() { + return isEnabled(Level.DEBUG); + } + + public boolean isInfoEnabled() { + return isEnabled(Level.INFO); + } + + public boolean isWarnEnabled() { + return isEnabled(Level.WARN); + } + + public boolean isErrorEnabled() { + return isEnabled(Level.ERROR); + } + + public void trace(String message, Object... args) { + log(Level.TRACE, message, args); + } + + public void debug(String message, Object... args) { + log(Level.DEBUG, message, args); + } + + public void info(String message, Object... args) { + log(Level.INFO, message, args); + } + + public void warn(String message, Object... args) { + log(Level.WARN, message, args); + } + + public void error(String message, Object... args) { + log(Level.ERROR, message, args); + } + + /// Writes `message` at `level`, its `{}` placeholders filled from `args`. + public void log(Level level, String message, Object... args) { + if (!isEnabled(level)) { + return; + } + StringBuilder line = new StringBuilder(64 + (message == null ? 4 : message.length())); + Timestamps.appendIso(line, System.currentTimeMillis()); + String label = level.name(); + line.append(' ').append(label); + for (int i = label.length(); i < 5; i++) { + line.append(' '); + } + line.append(" [").append(Thread.currentThread().getName()).append("] ").append(name).append(" - "); + int used = format(line, message, args); + Throwable error = null; + if (args != null && used < args.length && args[args.length - 1] instanceof Throwable) { + error = (Throwable) args[args.length - 1]; + } + MDC.appendTo(line); + Logging.write(level, name, line.toString(), error); + } + + private Level effectiveLevel() { + int current = Logging.generation(); + Level cached = level; + if (cached == null || generation != current) { + cached = Logging.levelOf(name); + level = cached; + generation = current; + } + return cached; + } + + /// Appends `message` with each `{}` replaced by the next argument, and + /// answers how many arguments it used. + static int format(StringBuilder out, String message, Object[] args) { + if (message == null) { + out.append("null"); + return 0; + } + int used = 0; + int count = args == null ? 0 : args.length; + int length = message.length(); + for (int i = 0; i < length; i++) { + char c = message.charAt(i); + if (c == '\\' && i + 2 < length && message.charAt(i + 1) == '{' && message.charAt(i + 2) == '}') { + out.append("{}"); + i += 2; + } else if (c == '{' && i + 1 < length && message.charAt(i + 1) == '}' && used < count) { + appendArgument(out, args[used++]); + i++; + } else { + out.append(c); + } + } + return used; + } + + private static void appendArgument(StringBuilder out, Object value) { + if (value instanceof Object[]) { + Object[] array = (Object[]) value; + out.append('['); + for (int i = 0; i < array.length; i++) { + if (i > 0) { + out.append(", "); + } + out.append(array[i]); + } + out.append(']'); + } else { + out.append(value); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/log/Logging.java b/vm/backend/src/com/codename1/backend/log/Logging.java new file mode 100644 index 00000000000..f1818003ab4 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/Logging.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +import com.codename1.backend.Config; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/// The logging configuration: per-logger levels and where lines go. +/// +/// Levels come from `logging.level.` settings, the keys Spring Boot uses. +/// A logger takes the level of the longest name that is it or a package above +/// it, and `logging.level.root` (INFO when unset) covers everything else: +/// +/// ```properties +/// logging.level.root=WARN +/// logging.level.com.example=INFO +/// logging.level.com.example.push=DEBUG +/// ``` +/// +/// The server applies its configuration when it starts; [#setLevel(String, Level)] +/// changes one at run time, and every logger sees the change on its next line. +public final class Logging { + /// The prefix of the per-logger level settings. + public static final String LEVEL_PREFIX = "logging.level."; + + /// Name/level pairs, longest name first, then the root level last. + private static final AtomicReference RULES = new AtomicReference( + new Object[] {"", Level.INFO}); + /// Bumped on every change, so a logger knows its cached level is stale. + private static final AtomicInteger GENERATION = new AtomicInteger(); + private static final AtomicReference SINK = new AtomicReference(new StandardSink(System.out)); + + private Logging() { + } + + /// Reads every `logging.level.*` setting of `config`, replacing the levels + /// set before. Called by the server when it starts. + /// + /// @throws IOException when a setting names no level + public static void configure(Config config) throws IOException { + List rules = new ArrayList(); + Level root = Level.INFO; + for (Object key : config.keys()) { + String name = String.valueOf(key); + if (!name.startsWith(LEVEL_PREFIX)) { + continue; + } + String logger = name.substring(LEVEL_PREFIX.length()); + String value = config.get(name, null); + Level level = Level.parse(value); + if (level == null) { + throw new IOException(name + "=" + value + " names no level; use TRACE, DEBUG, INFO, WARN, ERROR or OFF"); + } + if ("root".equals(logger)) { + root = level; + } else { + rules.add(logger); + rules.add(level); + } + } + install(rules, root); + } + + /// Sets the level of `logger` and every logger under it; null clears a level + /// set before. `"root"` is the level of everything else. + public static void setLevel(String logger, Level level) { + while (true) { + Object[] current = RULES.get(); + List rules = new ArrayList(); + Level root = (Level) current[current.length - 1]; + for (int i = 0; i + 2 < current.length; i += 2) { + if (!current[i].equals(logger)) { + rules.add(current[i]); + rules.add(current[i + 1]); + } + } + if ("root".equals(logger)) { + root = level == null ? Level.INFO : level; + } else if (level != null) { + rules.add(logger); + rules.add(level); + } + if (RULES.compareAndSet(current, sorted(rules, root))) { + GENERATION.incrementAndGet(); + return; + } + } + } + + /// Sends every line to `sink` from now on; null restores standard output. + public static void setSink(LogSink sink) { + SINK.set(sink == null ? new StandardSink(System.out) : sink); + } + + private static void install(List rules, Level root) { + RULES.set(sorted(rules, root)); + GENERATION.incrementAndGet(); + } + + /// The pairs, longest name first, so the first match is the most specific, + /// with the root level as the last element. + private static Object[] sorted(List rules, Level root) { + int pairs = rules.size() / 2; + String[] names = new String[pairs]; + Level[] levels = new Level[pairs]; + for (int i = 0; i < pairs; i++) { + names[i] = (String) rules.get(i * 2); + levels[i] = (Level) rules.get(i * 2 + 1); + } + for (int i = 1; i < pairs; i++) { + for (int j = i; j > 0 && names[j].length() > names[j - 1].length(); j--) { + String n = names[j]; + names[j] = names[j - 1]; + names[j - 1] = n; + Level l = levels[j]; + levels[j] = levels[j - 1]; + levels[j - 1] = l; + } + } + Object[] out = new Object[pairs * 2 + 1]; + for (int i = 0; i < pairs; i++) { + out[i * 2] = names[i]; + out[i * 2 + 1] = levels[i]; + } + out[pairs * 2] = root; + return out; + } + + static int generation() { + return GENERATION.get(); + } + + /// The level `logger` is set to: that of the longest configured name equal + /// to it or a dotted prefix of it, else the root level. + static Level levelOf(String logger) { + Object[] rules = RULES.get(); + for (int i = 0; i + 2 < rules.length; i += 2) { + String name = (String) rules[i]; + if (logger.startsWith(name) && (logger.length() == name.length() || logger.charAt(name.length()) == '.' + || logger.charAt(name.length()) == '$')) { + return (Level) rules[i + 1]; + } + } + return (Level) rules[rules.length - 1]; + } + + static void write(Level level, String logger, String line, Throwable error) { + SINK.get().write(level, logger, line, error); + } + + /// Writes each line, and the stack trace after it, to one stream. + private static final class StandardSink implements LogSink { + private final PrintStream out; + + StandardSink(PrintStream out) { + this.out = out; + } + + @Override + public void write(Level level, String logger, String line, Throwable error) { + // One println, so lines from two threads never interleave mid-line. + out.println(line); + if (error != null) { + error.printStackTrace(out); + } + } + } +} diff --git a/vm/backend/src/com/codename1/backend/log/MDC.java b/vm/backend/src/com/codename1/backend/log/MDC.java new file mode 100644 index 00000000000..1df0a43578d --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/MDC.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; + +/// A per-thread context map appended to every line logged on that thread, as +/// slf4j's MDC is: put a request id once and every line of that request carries +/// it. +/// +/// ```java +/// MDC.put("request", requestId); +/// try { +/// handle(request); +/// } finally { +/// MDC.remove("request"); +/// } +/// ``` +/// +/// The map belongs to the thread, a virtual thread included, so a value put by +/// one request is never seen by another. Remove what you put: a pooled thread +/// otherwise carries it into the next request it serves. +public final class MDC { + private static final ThreadLocal> CONTEXT = new ThreadLocal>(); + + private MDC() { + } + + /// Sets `key` for every line this thread logs from now on; a null value + /// removes it. + public static void put(String key, String value) { + if (key == null) { + throw new IllegalArgumentException("key is null"); + } + if (value == null) { + remove(key); + return; + } + TreeMap map = CONTEXT.get(); + if (map == null) { + map = new TreeMap(); + CONTEXT.set(map); + } + map.put(key, value); + } + + /// The value this thread set for `key`, or null. + public static String get(String key) { + TreeMap map = CONTEXT.get(); + return map == null ? null : map.get(key); + } + + /// Removes `key` from this thread's context. + public static void remove(String key) { + TreeMap map = CONTEXT.get(); + if (map != null) { + map.remove(key); + if (map.isEmpty()) { + CONTEXT.set(null); + } + } + } + + /// Empties this thread's context. + public static void clear() { + CONTEXT.set(null); + } + + /// Appends this thread's context as ` {key=value, ...}`, or nothing when empty. + static void appendTo(StringBuilder out) { + TreeMap map = CONTEXT.get(); + if (map == null || map.isEmpty()) { + return; + } + out.append(" {"); + boolean first = true; + for (Iterator> it = map.entrySet().iterator(); it.hasNext();) { + Map.Entry e = it.next(); + if (!first) { + out.append(", "); + } + first = false; + out.append(e.getKey()).append('=').append(e.getValue()); + } + out.append('}'); + } +} diff --git a/vm/backend/src/com/codename1/backend/log/Timestamps.java b/vm/backend/src/com/codename1/backend/log/Timestamps.java new file mode 100644 index 00000000000..b14177c4188 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/Timestamps.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.log; + +/// ISO-8601 UTC timestamps without java.time or a formatter. +final class Timestamps { + private Timestamps() { + } + + /// Appends `millis` as `yyyy-MM-ddTHH:mm:ss.SSSZ`. + static void appendIso(StringBuilder out, long millis) { + long days = Math.floorDiv(millis, 86400000L); + long ofDay = millis - days * 86400000L; + // Civil date from days since the epoch (H. Hinnant's days_from_civil inverse). + long z = days + 719468; + long era = Math.floorDiv(z, 146097); + long doe = z - era * 146097; + long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + long mp = (5 * doy + 2) / 153; + long day = doy - (153 * mp + 2) / 5 + 1; + long month = mp < 10 ? mp + 3 : mp - 9; + long year = yoe + era * 400 + (month <= 2 ? 1 : 0); + pad(out, year, 4).append('-'); + pad(out, month, 2).append('-'); + pad(out, day, 2).append('T'); + pad(out, ofDay / 3600000L, 2).append(':'); + pad(out, ofDay / 60000L % 60, 2).append(':'); + pad(out, ofDay / 1000L % 60, 2).append('.'); + pad(out, ofDay % 1000L, 3).append('Z'); + } + + private static StringBuilder pad(StringBuilder out, long value, int width) { + String digits = Long.toString(value); + for (int i = digits.length(); i < width; i++) { + out.append('0'); + } + return out.append(digits); + } +} diff --git a/vm/backend/src/com/codename1/backend/log/package-info.java b/vm/backend/src/com/codename1/backend/log/package-info.java new file mode 100644 index 00000000000..48815c96ac5 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/log/package-info.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// Application logging: [Logger], in the shape slf4j's is, with levels set by +/// `logging.level.*` settings, a per-thread [MDC] context, and a replaceable +/// [LogSink]. +package com.codename1.backend.log; From b8ce394ba73a7a10d22fcf7f162a0a93340d567c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:09:55 +0200 Subject: [PATCH 05/25] Backend: typed Json.read/readList/convert and Json.write through the generated codecs The build finds the class literals the application hands to the typed Json methods, writes a codec for each, and the entry point installs a generated registry over them. Co-Authored-By: Claude Opus 5.5 --- .../maven/processors/BackendJsonCodecs.java | 83 ++++++ .../processors/BackendTestGenerator.java | 7 + .../maven/processors/JsonCallSites.java | 247 ++++++++++++++++++ .../RestControllerAnnotationProcessor.java | 12 + ...RestControllerAnnotationProcessorTest.java | 61 +++++ .../src/com/codename1/backend/Json.java | 168 ++++++++++++ .../com/codename1/impl/backend/JsonCodec.java | 29 ++ 7 files changed, 607 insertions(+) create mode 100644 maven/build-engine/src/main/java/com/codename1/maven/processors/JsonCallSites.java diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java index fe1ff0045e6..c8bba06b825 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java @@ -238,6 +238,34 @@ String checkRead(String javaType) { return check(javaType, true, new HashSet()); } + /// The generated class `Json.read(String, Class)`, `Json.convert` and + /// `Json.write(Object)` reach the application's codecs through. Generated + /// only for a build whose code calls one of them. + static final String REGISTRY = "cn1app.JsonCodecs"; + + /// Whether the application's code calls the typed `Json` methods. + private boolean registryWanted; + + /// A typed `Json` call in the application's code: `type`, a binary name or + /// null when the call site does not say, is read (`read`) or written by it. + /// Answers null, or why the build cannot write that codec. + String typedUse(String type, boolean read) { + registryWanted = true; + String t = type == null ? null : strip(type); + if (t == null || kindOf(t) != Kind.DTO) { + // A JDK type Json reads by itself, or one the call site does not + // name; only an application class needs a codec. + return null; + } + return read ? checkRead(t) : checkWrite(t); + } + + /// Whether [#sources] includes [#REGISTRY], for the entry point that has to + /// install it. + boolean hasRegistry() { + return registryWanted; + } + /// Whether any codec is needed at all. boolean isEmpty() { return dtos.isEmpty(); @@ -1681,6 +1709,10 @@ Map sources() { owners.put(codec, dto.key); out.put(codec, codecSource(dto)); } + if (registryWanted) { + valuesUsed = true; + out.put(REGISTRY, registrySource()); + } if (valuesUsed) { out.put(VALUES, valuesSource()); } @@ -1690,6 +1722,57 @@ Map sources() { return out; } + /// `cn1app.JsonCodecs`: [#REGISTRY]. Reads by `==` on the class asked for and + /// writes through `cn1app.JsonValues`, so a value written by `Json.write` + /// has the form a route answering with it has. + private String registrySource() { + StringBuilder sb = new StringBuilder(); + sb.append("package cn1app;\n\n"); + sb.append("// Generated: the codecs Json reaches by class. Do not edit.\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public final class JsonCodecs implements ").append(CODEC) + .append(".Registry {\n\n"); + sb.append(" @Override\n"); + sb.append(" public Object read(Class type, Object json) {\n"); + StringBuilder writes = new StringBuilder(); + // A copy: reading a class can ask for the codecs of the classes it holds. + for (Dto dto : new ArrayList(dtos.values())) { + // A generic class has a codec per use, and a Class names none of them. + if (dto.problem != null || !dto.key.equals(dto.binary)) { + continue; + } + String type = source(dto.binary); + if (dto.read && dto.readProblem == null) { + sb.append(" if (type == ").append(type).append(".class) {\n"); + sb.append(" ").append(type).append(" r;\n"); + sb.append(readStatements(dto.binary, "json", CODEC + ".Path.ROOT", "null", "-1", + "0", "r", " ")); + sb.append(" return r;\n"); + sb.append(" }\n"); + } + if (dto.write) { + writes.append(writes.length() == 0 ? "" : "\n || ") + .append("v instanceof ").append(type); + } + } + sb.append(" throw new IllegalStateException(\"No JSON codec for \" + type.getName()\n"); + sb.append(" + \": the build writes one for a class passed literally to \"\n"); + sb.append(" + \"Json.read, Json.readList or Json.convert, or used by a route.\");\n"); + sb.append(" }\n\n"); + sb.append(" @Override\n"); + sb.append(" public boolean write(Object v, ").append(SINK).append(" out) {\n"); + if (writes.length() > 0) { + sb.append(" if (").append(writes).append(") {\n"); + sb.append(" JsonValues.write(v, out, 0);\n"); + sb.append(" return true;\n"); + sb.append(" }\n"); + } + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append("}\n"); + return sb.toString(); + } + /// `cn1app.JsonValues`: writes a value by its run-time class. Each of the /// application's classes this build writes goes through its codec, the JDK's /// JSON shapes through Json, and anything else is refused with an exception diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendTestGenerator.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendTestGenerator.java index 8fa88496eee..23b70bffc4b 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendTestGenerator.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendTestGenerator.java @@ -809,6 +809,13 @@ private String context(Spec spec) { sb.append(" new ").append(OrmAnnotationProcessor.BACKEND_BOOTSTRAP_BINARY) .append("();\n"); } + String registry = BackendJsonCodecs.REGISTRY.replace('.', '/'); + if (ctx.lookup(registry) != null + || RestControllerAnnotationProcessor.resolveClass(ctx, registry) != null) { + // As the generated main installs it: Json.read(body, Release.class). + sb.append(" com.codename1.impl.backend.JsonCodec.install(new ") + .append(BackendJsonCodecs.REGISTRY).append("());\n"); + } sb.append(" }\n\n"); sb.append(" public com.codename1.impl.backend.BackendApplication createApplication() {\n") .append(" return new ").append(spec.wiringBinary).append("();\n }\n\n"); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/JsonCallSites.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/JsonCallSites.java new file mode 100644 index 00000000000..6bcd03bd960 --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/JsonCallSites.java @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.ProcessorContext; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +/// Finds the application's calls to the typed `Json` methods, so the build +/// writes a codec for each class they name. +/// +/// `Json.read(body, Release.class)` names its class with the literal right +/// before the call: the class is the last argument, so its `LDC` is the last +/// instruction before the `INVOKESTATIC`. `Json.write(release)` names nothing, +/// and the type of what is written is taken from the instruction that put it on +/// the stack -- a method's return type, a field's, a cast, or for a local what +/// was stored in it and its declared type in the debug table. A site that says nothing still makes the build +/// generate the registry, and a class some other use gave a codec works there. +final class JsonCallSites { + private static final String JSON = "com/codename1/backend/Json"; + + private JsonCallSites() { + } + + /// Hands every typed call site of the application's own classes to `codecs`, + /// and reports a class the build cannot write a codec for at its call. + static void scan(final ProcessorContext ctx, final BackendJsonCodecs codecs) { + for (final AnnotatedClass cls : new ArrayList( + ctx.getClassIndex().values())) { + File file = cls.getClassFile(); + if (file == null || !file.isFile() || ctx.isDependencyClass(cls) + || cls.isFromBackendLibrary()) { + continue; + } + byte[] bytes; + try { + bytes = Files.readAllBytes(file.toPath()); + } catch (IOException e) { + ctx.error(cls, "Cannot read " + file + ": " + e.getMessage()); + continue; + } + new ClassReader(bytes).accept(new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, final String name, String desc, + String signature, String[] exceptions) { + return new Sites(ctx, codecs, cls, name); + } + }, ClassReader.SKIP_FRAMES); + } + } + + /// One method: the type the last instruction left on the stack, and the + /// writes of a local whose type only the debug table at the end can say. + private static final class Sites extends MethodVisitor { + private final ProcessorContext ctx; + private final BackendJsonCodecs codecs; + private final AnnotatedClass cls; + private final String method; + /// The class the last instruction pushed, as a binary name, or null. + private String last; + /// The local the last instruction loaded, or -1. + private int lastLocal = -1; + private final List writtenLocals = new ArrayList(); + private final Map> localTypes = new HashMap>(); + + Sites(ProcessorContext ctx, BackendJsonCodecs codecs, AnnotatedClass cls, String method) { + super(Opcodes.ASM9); + this.ctx = ctx; + this.codecs = codecs; + this.cls = cls; + this.method = method; + } + + private void pushed(String type) { + last = type; + lastLocal = -1; + } + + @Override + public void visitLdcInsn(Object value) { + pushed(value instanceof Type && ((Type) value).getSort() == Type.OBJECT + ? ((Type) value).getClassName() : null); + } + + @Override + public void visitVarInsn(int opcode, int var) { + if (opcode == Opcodes.ASTORE && last != null) { + // What the slot was given: the type a class compiled without its + // debug table still says, through the cast that came before. + local(var, last); + } + pushed(null); + if (opcode == Opcodes.ALOAD) { + lastLocal = var; + } + } + + @Override + public void visitTypeInsn(int opcode, String type) { + pushed(opcode == Opcodes.CHECKCAST ? objectType(Type.getObjectType(type)) : null); + } + + @Override + public void visitFieldInsn(int opcode, String owner, String name, String desc) { + pushed(opcode == Opcodes.GETFIELD || opcode == Opcodes.GETSTATIC + ? objectType(Type.getType(desc)) : null); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String desc, + boolean itf) { + if (opcode == Opcodes.INVOKESTATIC && JSON.equals(owner)) { + if (("read".equals(name) || "readList".equals(name)) + && "(Ljava/lang/String;Ljava/lang/Class;)".equals(params(desc)) + || "convert".equals(name) + && "(Ljava/lang/Object;Ljava/lang/Class;)".equals(params(desc))) { + use(last, true, name); + } else if ("write".equals(name) + && "(Ljava/lang/Object;)Ljava/lang/String;".equals(desc)) { + if (lastLocal >= 0) { + writtenLocals.add(Integer.valueOf(lastLocal)); + } else if (last != null) { + use(last, false, name); + } + } + } + pushed(objectType(Type.getReturnType(desc))); + } + + @Override + public void visitInsn(int opcode) { + pushed(null); + } + + @Override + public void visitIntInsn(int opcode, int operand) { + pushed(null); + } + + @Override + public void visitJumpInsn(int opcode, Label label) { + pushed(null); + } + + @Override + public void visitLabel(Label label) { + pushed(null); + } + + @Override + public void visitInvokeDynamicInsn(String name, String desc, + org.objectweb.asm.Handle bsm, Object... args) { + pushed(null); + } + + @Override + public void visitLocalVariable(String name, String desc, String signature, Label start, + Label end, int index) { + String type = objectType(Type.getType(desc)); + if (type != null) { + local(index, type); + } + } + + private void local(int index, String type) { + List types = localTypes.get(Integer.valueOf(index)); + if (types == null) { + types = new ArrayList(); + localTypes.put(Integer.valueOf(index), types); + } + if (!types.contains(type)) { + types.add(type); + } + } + + @Override + public void visitEnd() { + for (Integer local : writtenLocals) { + List types = localTypes.get(local); + if (types != null) { + // A slot reused for another variable names every type it held; + // a codec too many is harmless. + for (String type : types) { + use(type, false, "write"); + } + } + } + } + + private void use(String type, boolean read, String call) { + if (!read && (type == null || type.startsWith("java."))) { + // Json writes the JDK's shapes itself, and an Object says nothing. + return; + } + String why = codecs.typedUse(type, read); + if (why != null) { + ctx.error(cls, "Json." + call + " in " + cls.getSourceName() + "." + method + + " " + (read ? "reads " : "writes ") + + BackendJsonCodecs.source(type) + ", which cannot be " + + (read ? "read" : "written") + " as JSON: " + why); + } + } + } + + private static String params(String desc) { + return desc.substring(0, desc.indexOf(')') + 1); + } + + /// The binary name of `type` when it is a class, else null. + private static String objectType(Type type) { + return type.getSort() == Type.OBJECT ? type.getClassName() : null; + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java index 581be9a85cf..7fb7bdb4b45 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -1665,6 +1665,12 @@ public void finish(ProcessorContext ctx) throws ProcessingException { + "would replace it. Rename that class."); return; } + // Json.read(body, Release.class) in the application's own code: each class + // it names gets a codec, as a route's body does. + JsonCallSites.scan(ctx, codecs(ctx)); + if (ctx.hasErrors()) { + return; + } // The shared set, not only this processor's: the bean processor's tools // may have added classes, and the dispatcher compiled last must know them. if (codecs != null || ctx.getAttribute("cn1.backend.jsonCodecs") != null) { @@ -3403,6 +3409,12 @@ String generateBootstrap(String packageName) { .append(quote(telemetryServiceName)).append(");\n"); } } + if (codecs != null && codecs.hasRegistry()) { + // The ONLY reference to the registry: Json.read(body, Release.class) + // reaches the application's codecs through it, and nothing else does. + sb.append(" com.codename1.impl.backend.JsonCodec.install(new ") + .append(BackendJsonCodecs.REGISTRY).append("());\n"); + } sb.append(" cn1Access.application(cn1Builder, new ") .append(qualify(packageName, BackendWiringWriter.CLASS_NAME)).append("());\n"); sb.append(" cn1Builder.run();\n"); diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java index 5bd2b0165d2..a6bee8562ab 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java @@ -1554,6 +1554,67 @@ public void aPlainClassIsWrittenByItsCodec() throws Exception { assertFalse(ctx.getErrors().toString(), ctx.hasErrors()); } + @Test + public void jsonReadAndWriteReachTheCodecOfAClassNamedInCode() throws Exception { + // Release is used by no route: only the Json.read call names it, and the + // build has to find that call to write its codec. + Map sources = new LinkedHashMap(); + sources.put("com.example.Release", + "package com.example;\n" + + "public class Release {\n" + + " public String name;\n" + + " public int build;\n" + + "}\n"); + sources.put("com.example.Feed", + "package com.example;\n" + + "import com.codename1.backend.Json;\n" + + "public class Feed {\n" + + " public static String roundTrip(String body) throws java.io.IOException {\n" + + " Release r = Json.read(body, Release.class);\n" + + " r.build++;\n" + + " return Json.write(r);\n" + + " }\n" + + " public static int count(String body) throws java.io.IOException {\n" + + " return Json.readList(body, Release.class).size();\n" + + " }\n" + + "}\n"); + sources.put("com.example.Notes", + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/n\")\n" + + " public String n() { return \"n\"; }\n" + + "}\n"); + File classes = tmp.newFolder(); + JavaSourceCompiler.compile(sources, classes, backendClasspath()); + ProcessorContext ctx = run(classes); + assertFalse(ctx.getErrors().toString(), ctx.hasErrors()); + URLClassLoader loader = new URLClassLoader(new URL[]{ classes.toURI().toURL() }, + getClass().getClassLoader()); + com.codename1.impl.backend.JsonCodec.install((com.codename1.impl.backend.JsonCodec.Registry) + loader.loadClass("cn1app.JsonCodecs").newInstance()); + try { + Class feed = loader.loadClass("com.example.Feed"); + assertEquals("{\"name\":\"v1\",\"build\":8}", feed.getMethod("roundTrip", + String.class).invoke(null, "{\"name\":\"v1\",\"build\":7}")); + assertEquals(2, feed.getMethod("count", String.class).invoke(null, + "[{\"name\":\"a\"},{\"name\":\"b\"}]")); + try { + feed.getMethod("roundTrip", String.class).invoke(null, "{\"build\":\"x\"}"); + fail("a string where the class has an int must be refused"); + } catch (java.lang.reflect.InvocationTargetException e) { + assertTrue(String.valueOf(e.getCause()), + e.getCause() instanceof IllegalArgumentException + && e.getCause().getMessage().indexOf("build") >= 0); + } + assertEquals(Long.valueOf(3), com.codename1.backend.Json.read("3", Long.class)); + assertEquals(Integer.valueOf(3), com.codename1.backend.Json.read("3", Integer.class)); + } finally { + com.codename1.impl.backend.JsonCodec.install(null); + } + } + @Test public void aParameterWithTwoBindingAnnotationsIsRefused() throws Exception { // The binding chain is priority-ordered, so this used to bind the header diff --git a/vm/backend/src/com/codename1/backend/Json.java b/vm/backend/src/com/codename1/backend/Json.java index 6ca2ea8ccbc..05c4c972cc8 100644 --- a/vm/backend/src/com/codename1/backend/Json.java +++ b/vm/backend/src/com/codename1/backend/Json.java @@ -22,8 +22,10 @@ */ package com.codename1.backend; +import com.codename1.impl.backend.JsonCodec; import com.codename1.io.JSONParser; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; @@ -61,6 +63,151 @@ public static Object parse(String json) throws IOException { return JSONParser.parseStrict(json); } + /// Reads a JSON document into `type`, as Jackson's `readValue` does. + /// + /// ```java + /// Release release = Json.read(body, Release.class); + /// ``` + /// + /// An application class is read by the codec the build generated for it -- + /// the one a route taking it as a `@RequestBody` uses, so `@JsonProperty`, + /// `@JsonIgnore` and the rest mean the same thing here. The build writes a + /// codec for every class literal passed to this method, [#readList] and + /// [#convert] in the application's code, so name the class literally: a + /// `Class` that arrives in a variable is read only if some other use made the + /// build write its codec, and is refused with `IllegalStateException` + /// otherwise. + /// + /// `String`, the boxed numbers, `Boolean`, `Map`, `List` and `Object` need + /// no codec. + /// + /// @param json the document + /// @param type what to read it into + /// @return the value, or null for a JSON `null` + /// @throws IOException when `json` is not JSON + /// @throws IllegalArgumentException when the document does not fit `type`; + /// the message names where, as `$.items[2].due` + public static T read(String json, Class type) throws IOException { + return convert(parse(json), type); + } + + /// Reads a JSON array of `element`, as Jackson's + /// `readValue(json, new TypeReference>() {})` does. + /// + /// @param json the document, an array + /// @param element what each element is read into; see [#read(String, Class)] + /// @return the elements, in order + /// @throws IOException when `json` is not JSON + /// @throws IllegalArgumentException when it is not an array, or an element + /// does not fit `element` + public static List readList(String json, Class element) throws IOException { + Object tree = parse(json); + if (!(tree instanceof List)) { + throw new IllegalArgumentException("$: expected an array, got " + describe(tree)); + } + List in = (List) tree; + List out = new ArrayList(in.size()); + for (int iter = 0 ; iter < in.size() ; iter++) { + out.add(convert(in.get(iter), element)); + } + return out; + } + + /// Reads a document already parsed by [#parse] -- or a member of one -- into + /// `type`, as Jackson's `convertValue` does. See [#read(String, Class)]. + /// + /// @param tree what [#parse] returned, or part of it + /// @param type what to read it into + /// @return the value, or null for null + /// @throws IllegalArgumentException when `tree` does not fit `type` + public static T convert(Object tree, Class type) { + if (type == null) { + throw new IllegalArgumentException("type is null"); + } + if (tree == null) { + return null; + } + Object value = builtIn(tree, type); + if (value == null) { + JsonCodec.Registry codecs = JsonCodec.registry(); + if (codecs == null) { + throw new IllegalStateException("No JSON codec for " + type.getName() + + ": this binary has none. The build writes one for a class passed " + + "literally to Json.read, Json.readList or Json.convert, or used by " + + "a route; nothing reads a class by reflection."); + } + value = codecs.read(type, tree); + } + // Checked rather than cast: the translated runtime does not check a cast. + if (value != null && !type.isInstance(value)) { + throw new IllegalArgumentException("$: expected " + type.getName() + ", got " + + describe(tree)); + } + return (T) value; + } + + /// `tree` as one of the types that need no codec, or null when `type` is not + /// one of them. + private static Object builtIn(Object tree, Class type) { + if (type == Object.class) { + return tree; + } + if (type == String.class || type == Boolean.class || type == Map.class + || type == List.class) { + if (!type.isInstance(tree)) { + throw new IllegalArgumentException("$: expected " + type.getName() + ", got " + + describe(tree)); + } + return tree; + } + if (type == Long.class || type == Integer.class || type == Double.class + || type == Float.class || type == Short.class || type == Byte.class) { + if (!(tree instanceof Number)) { + throw new IllegalArgumentException("$: expected a number, got " + + describe(tree)); + } + Number n = (Number) tree; + if (type == Double.class) { + return Double.valueOf(n.doubleValue()); + } + if (type == Float.class) { + return Float.valueOf(n.floatValue()); + } + if (tree instanceof Double && n.doubleValue() != Math.floor(n.doubleValue())) { + throw new IllegalArgumentException("$: expected a whole number, got " + tree); + } + long l = n.longValue(); + if (type == Long.class) { + return Long.valueOf(l); + } + long min = type == Integer.class ? Integer.MIN_VALUE + : type == Short.class ? Short.MIN_VALUE : Byte.MIN_VALUE; + long max = type == Integer.class ? Integer.MAX_VALUE + : type == Short.class ? Short.MAX_VALUE : Byte.MAX_VALUE; + if (l < min || l > max) { + throw new IllegalArgumentException("$: " + l + " is out of range for " + + type.getName()); + } + return type == Integer.class ? (Object) Integer.valueOf((int) l) + : type == Short.class ? (Object) Short.valueOf((short) l) + : (Object) Byte.valueOf((byte) l); + } + return null; + } + + private static String describe(Object tree) { + if (tree instanceof Map) { + return "an object"; + } + if (tree instanceof List) { + return "an array"; + } + if (tree instanceof String) { + return "a string"; + } + return String.valueOf(tree); + } + // ------------------------------------------------------------------ // Writing // ------------------------------------------------------------------ @@ -226,6 +373,12 @@ private static void writeValue(ByteSink out, Object value) { writeString(out, Base64Url.encode((byte[]) value)); return; } + JsonCodec.Registry codecs = JsonCodec.registry(); + if (codecs != null && codecs.write(value, out)) { + // One of the application's own classes, through the codec the build + // wrote for it -- the same one a route answering with it uses. + return; + } writeString(out, String.valueOf(value)); } @@ -388,6 +541,21 @@ private static void writeValue(StringBuilder out, Object value) { out.append(']'); return; } + JsonCodec.Registry codecs = JsonCodec.registry(); + if (codecs != null) { + // As the sink writer does: the two have to agree on an application + // class, or it is an object over one protocol and a string over the + // other. + ByteSink sink = new ByteSink(256); + if (codecs.write(value, sink)) { + try { + out.append(new String(sink.bytes(), 0, sink.length(), "UTF-8")); + } catch (java.io.UnsupportedEncodingException never) { + throw new IllegalStateException(never.getMessage()); + } + return; + } + } writeString(out, value.toString()); } diff --git a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java index ccfbd256c24..169e861f39b 100644 --- a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java +++ b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java @@ -27,6 +27,7 @@ import com.codename1.backend.ByteSink; import java.util.Date; +import java.util.concurrent.atomic.AtomicReference; /// What the JSON codecs the build generates call: typed reads that refuse a /// value with a message naming where it was, and the date form. @@ -55,6 +56,34 @@ public final class JsonCodec { private JsonCodec() { } + private static final AtomicReference REGISTRY = new AtomicReference(); + + /// The codecs `Json.read(String, Class)` and `Json.write(Object)` reach a + /// class of the application's through. The build generates the one + /// implementation, `cn1app.JsonCodecs`, over every class it wrote a codec + /// for, and the generated entry point installs it: nothing is found by name. + public interface Registry { + /// `json`, a parsed document, read into `type`. Throws + /// `IllegalArgumentException` naming where the document does not fit, + /// and `IllegalStateException` when the build wrote no codec for `type`. + Object read(Class type, Object json); + + /// Writes `value` when the build wrote a codec for its class, and + /// answers whether it did. + boolean write(Object value, ByteSink out); + } + + /// Installs the build's registry; the generated entry point calls it once, + /// before the server starts. + public static void install(Registry registry) { + REGISTRY.set(registry); + } + + /// The installed registry, or null in a binary whose build wrote none. + public static Registry registry() { + return REGISTRY.get(); + } + /// How deep the codecs follow one object into another: [#MAX_DEPTH] unless /// `cn1.json.maxDepth` says otherwise. public static int maxDepth() { From c0cecbaf18ea37aab4135ccabc165e16df844d04 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:10:09 +0200 Subject: [PATCH 06/25] Backend: XML pull parser, element tree, property lists and image header reader (#6153 items 6, 7) com.codename1.backend.xml: XmlPullParser (XXE-safe by construction: DOCTYPE skipped, only predefined and numeric entities, nesting capped), XmlElement, PropertyLists (XML plists and the plist inside a signed provisioning profile). com.codename1.backend.ImageInfo: PNG/JPEG/GIF/WebP format and dimensions from the header, JPEG walked segment by segment to its SOFn. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/backend/ImageInfoTest.java | 149 +++++ .../backend/xml/PropertyListsTest.java | 137 +++++ .../src/com/codename1/backend/ImageInfo.java | 312 +++++++++++ .../codename1/backend/xml/PropertyLists.java | 296 ++++++++++ .../com/codename1/backend/xml/XmlElement.java | 186 +++++++ .../codename1/backend/xml/XmlException.java | 45 ++ .../codename1/backend/xml/XmlPullParser.java | 516 ++++++++++++++++++ .../codename1/backend/xml/package-info.java | 35 ++ 8 files changed, 1676 insertions(+) create mode 100644 maven/backend/src/test/java/com/codename1/backend/ImageInfoTest.java create mode 100644 maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java create mode 100644 vm/backend/src/com/codename1/backend/ImageInfo.java create mode 100644 vm/backend/src/com/codename1/backend/xml/PropertyLists.java create mode 100644 vm/backend/src/com/codename1/backend/xml/XmlElement.java create mode 100644 vm/backend/src/com/codename1/backend/xml/XmlException.java create mode 100644 vm/backend/src/com/codename1/backend/xml/XmlPullParser.java create mode 100644 vm/backend/src/com/codename1/backend/xml/package-info.java diff --git a/maven/backend/src/test/java/com/codename1/backend/ImageInfoTest.java b/maven/backend/src/test/java/com/codename1/backend/ImageInfoTest.java new file mode 100644 index 00000000000..5b2666e90f8 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/ImageInfoTest.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ImageInfoTest { + private static byte[] png(int w, int h, int depth, int type, boolean trns) { + ByteArrayOutputStream o = new ByteArrayOutputStream(); + o.write(new byte[] {(byte) 0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, 0, 8); + chunk(o, "IHDR", new byte[] {(byte) (w >> 24), (byte) (w >> 16), (byte) (w >> 8), (byte) w, + (byte) (h >> 24), (byte) (h >> 16), (byte) (h >> 8), (byte) h, (byte) depth, (byte) type, 0, 0, 0}); + if (trns) { + chunk(o, "tRNS", new byte[] {0}); + } + chunk(o, "IDAT", new byte[4]); + chunk(o, "IEND", new byte[0]); + return o.toByteArray(); + } + + private static void chunk(ByteArrayOutputStream o, String type, byte[] body) { + int n = body.length; + o.write(new byte[] {(byte) (n >> 24), (byte) (n >> 16), (byte) (n >> 8), (byte) n}, 0, 4); + for (int i = 0; i < 4; i++) { + o.write(type.charAt(i)); + } + o.write(body, 0, n); + o.write(new byte[4], 0, 4); + } + + private static void segment(ByteArrayOutputStream o, int marker, byte[] body) { + o.write(0xff); + o.write(marker); + int n = body.length + 2; + o.write(n >> 8); + o.write(n); + o.write(body, 0, body.length); + } + + private static byte[] jpeg(int sof, int w, int h) { + ByteArrayOutputStream o = new ByteArrayOutputStream(); + o.write(0xff); + o.write(0xd8); + segment(o, 0xe0, "JFIF\0\1\1\0\0\1\0\1\0\0".getBytes()); + byte[] exif = new byte[3000]; + exif[0] = 'E'; + exif[1] = 'x'; + exif[2] = 'i'; + exif[3] = 'f'; + exif[100] = (byte) 0xff; + exif[101] = (byte) 0xc0; + segment(o, 0xe1, exif); + segment(o, 0xdb, new byte[65]); + segment(o, 0xc4, new byte[20]); + o.write(0xff); // fill byte + segment(o, sof, new byte[] {8, (byte) (h >> 8), (byte) h, (byte) (w >> 8), (byte) w, 3, 1, 0x22, 0, + 2, 0x11, 1, 3, 0x11, 1}); + segment(o, 0xda, new byte[10]); + o.write(0xff); + o.write(0xd9); + return o.toByteArray(); + } + + @Test + void png() throws Exception { + ImageInfo i = ImageInfo.read(png(1290, 2796, 8, 6, false)); + assertEquals(ImageInfo.Format.PNG, i.getFormat()); + assertEquals(1290, i.getWidth()); + assertEquals(2796, i.getHeight()); + assertEquals(8, i.getBitDepth()); + assertEquals(6, i.getColorType()); + assertTrue(i.hasAlpha()); + assertFalse(ImageInfo.read(png(10, 20, 8, 2, false)).hasAlpha()); + assertTrue(ImageInfo.read(new ByteArrayInputStream(png(10, 20, 8, 3, true))).hasAlpha()); + } + + @Test + void baselineJpegAfterExif() { + ImageInfo i = ImageInfo.read(jpeg(0xc0, 2048, 1536)); + assertEquals(ImageInfo.Format.JPEG, i.getFormat()); + assertEquals(2048, i.getWidth()); + assertEquals(1536, i.getHeight()); + assertEquals(8, i.getBitDepth()); + assertFalse(i.isProgressive()); + } + + @Test + void progressiveJpeg() { + ImageInfo i = ImageInfo.read(jpeg(0xc2, 640, 480)); + assertEquals(640, i.getWidth()); + assertEquals(480, i.getHeight()); + assertTrue(i.isProgressive()); + } + + @Test + void gifAndWebp() { + byte[] gif = {'G', 'I', 'F', '8', '9', 'a', 0x40, 0x01, (byte) 0xf0, 0x00, 0, 0, 0, 0}; + ImageInfo g = ImageInfo.read(gif); + assertEquals(ImageInfo.Format.GIF, g.getFormat()); + assertEquals(320, g.getWidth()); + assertEquals(240, g.getHeight()); + byte[] webp = new byte[30]; + System.arraycopy("RIFF\0\0\0\0WEBPVP8X".getBytes(), 0, webp, 0, 16); + webp[20] = 0x10; + webp[24] = 99; // width - 1 + webp[27] = 49; // height - 1 + ImageInfo w = ImageInfo.read(webp); + assertEquals(ImageInfo.Format.WEBP, w.getFormat()); + assertEquals(100, w.getWidth()); + assertEquals(50, w.getHeight()); + assertTrue(w.hasAlpha()); + } + + @Test + void unrecognizedOrTruncated() { + assertNull(ImageInfo.read(new byte[] {1, 2, 3})); + assertNull(ImageInfo.read("hello, this is not an image".getBytes())); + byte[] cut = jpeg(0xc0, 10, 10); + assertNull(ImageInfo.read(java.util.Arrays.copyOf(cut, 40))); + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java new file mode 100644 index 00000000000..b62281d9bb9 --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.xml; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PropertyListsTest { + private static final String PLIST = "\n" + + "\n" + + "\n\n" + + " AppIDNameMy & App A\n" + + " TeamIdentifierABCDE12345\n" + + " ExpirationDate2025-01-31T12:30:15Z\n" + + " TimeToLive365\n" + + " Ratio-1.5\n" + + " Yes\n No\n" + + " Cert\n AQID\n BA==\n \n" + + " Entitlementsget-task-allow" + + "empty\n" + + " Note c]]>\n" + + "\n\n"; + + @SuppressWarnings("unchecked") + @Test + void parsesEveryValueType() throws Exception { + Map m = (Map) PropertyLists.parse(PLIST.getBytes(StandardCharsets.UTF_8)); + assertEquals("My & App A", m.get("AppIDName")); + assertEquals("ABCDE12345", ((List) m.get("TeamIdentifier")).get(0)); + assertEquals(new Date(1738326615000L), m.get("ExpirationDate")); + assertEquals(Long.valueOf(365), m.get("TimeToLive")); + assertEquals(Double.valueOf(-1.5), m.get("Ratio")); + assertEquals(Boolean.TRUE, m.get("Yes")); + assertEquals(Boolean.FALSE, m.get("No")); + assertArrayEquals(new byte[] {1, 2, 3, 4}, (byte[]) m.get("Cert")); + Map ent = (Map) m.get("Entitlements"); + assertEquals(Boolean.FALSE, ent.get("get-task-allow")); + assertTrue(((List) ent.get("empty")).isEmpty()); + assertEquals("a c", m.get("Note")); + } + + @SuppressWarnings("unchecked") + @Test + void findsThePlistInsideASignedProfile() throws Exception { + byte[] xml = PLIST.getBytes(StandardCharsets.UTF_8); + byte[] blob = new byte[xml.length + 600]; + for (int i = 0; i < blob.length; i++) { + blob[i] = (byte) (i * 31 + 7); + } + blob[0] = 0x30; + blob[1] = (byte) 0x82; + System.arraycopy(xml, 0, blob, 300, xml.length); + byte[] span = PropertyLists.extractXml(blob); + assertTrue(new String(span, StandardCharsets.UTF_8).startsWith("")); + Map m = (Map) PropertyLists.parse(blob); + assertEquals(Long.valueOf(365), m.get("TimeToLive")); + assertNull(PropertyLists.extractXml(new byte[] {1, 2, 3})); + } + + @Test + void externalEntityIsNeverExpanded() { + String xxe = "" + + "\">]>&xxe;"; + XmlException e = assertThrows(XmlException.class, () -> PropertyLists.parse(xxe)); + assertTrue(e.getMessage().contains("undefined entity"), e.getMessage()); + } + + @Test + void deepNestingIsRefused() { + StringBuilder b = new StringBuilder(""); + for (int i = 0; i < 10000; i++) { + b.append(""); + } + XmlException e = assertThrows(XmlException.class, () -> PropertyLists.parse(b.toString())); + assertTrue(e.getMessage().contains("nested deeper"), e.getMessage()); + } + + @Test + void malformedInputIsAnXmlException() { + assertThrows(XmlException.class, () -> PropertyLists.parse("a")); + assertThrows(XmlException.class, () -> PropertyLists.parse("x")); + assertThrows(XmlException.class, () -> PropertyLists.parse("a")); + assertThrows(XmlException.class, () -> PropertyLists.parse("bplist00xyz".getBytes(StandardCharsets.UTF_8))); + } + + @Test + void pullParserEvents() throws Exception { + XmlPullParser p = XmlPullParser.of("t"); + assertEquals(XmlPullParser.START_ELEMENT, p.next()); + assertEquals("a", p.getName()); + assertEquals("1", p.getAttributeValue("x")); + assertEquals("<", p.getAttributeValue("y")); + assertEquals(XmlPullParser.START_ELEMENT, p.next()); + assertEquals(XmlPullParser.END_ELEMENT, p.next()); + assertEquals("b", p.getName()); + assertEquals(XmlPullParser.TEXT, p.next()); + assertEquals("t", p.getText()); + assertEquals(XmlPullParser.END_ELEMENT, p.next()); + assertEquals(XmlPullParser.END_DOCUMENT, p.next()); + XmlElement root = XmlElement.parse("v"); + assertEquals(2, root.children("i").size()); + assertEquals("v", root.childText("i").isEmpty() ? root.children("i").get(1).getText() : "?"); + assertEquals("1", root.child("i").getAttribute("n")); + } +} diff --git a/vm/backend/src/com/codename1/backend/ImageInfo.java b/vm/backend/src/com/codename1/backend/ImageInfo.java new file mode 100644 index 00000000000..034db0b37ee --- /dev/null +++ b/vm/backend/src/com/codename1/backend/ImageInfo.java @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +/// The format and dimensions of an image, read from its header without decoding +/// any pixels. +/// +/// ```java +/// ImageInfo info = ImageInfo.read(upload); +/// if (info == null || info.getFormat() != ImageInfo.Format.PNG) { +/// throw new ResponseStatusException(400, "a PNG is required"); +/// } +/// if (info.getWidth() != 1290 || info.getHeight() != 2796) { +/// throw new ResponseStatusException(400, "wrong screenshot size"); +/// } +/// ``` +/// +/// PNG, JPEG, GIF and WebP are recognized by their signature, never by a file +/// name. The answer is what the header *claims*: a file with a valid header and a +/// corrupt body still reads, so this checks an upload's shape, not that it will +/// display. +/// +/// - **PNG**: width and height from `IHDR`, plus [#getBitDepth()], +/// [#getColorType()] and [#hasAlpha()] (an alpha color type, or a `tRNS` +/// chunk before the image data). +/// - **JPEG**: the segments are walked to the first start-of-frame marker +/// (`SOF0`-`SOF15`, excluding `DHT`, `JPG` and `DAC`), skipping `APPn` +/// segments such as EXIF and fill bytes; [#isProgressive()] reports a +/// progressive frame and [#getBitDepth()] the sample precision. EXIF orientation +/// is not applied: the dimensions are the stored ones. +/// - **GIF**: the logical screen size. +/// - **WebP**: lossy (`VP8 `), lossless (`VP8L`) and extended (`VP8X`), with +/// [#hasAlpha()]. +/// +/// Anything else answers null. +public final class ImageInfo { + /// An image format this class recognizes. + public enum Format { + /// Portable Network Graphics. + PNG, + /// JPEG / JFIF / EXIF. + JPEG, + /// Graphics Interchange Format. + GIF, + /// WebP, lossy or lossless. + WEBP + } + + private static final int MAX_STREAM_BYTES = 64 * 1024 * 1024; + + private final Format format; + private final int width; + private final int height; + private final int bitDepth; + private final int colorType; + private final boolean alpha; + private final boolean progressive; + + private ImageInfo(Format format, int width, int height, int bitDepth, int colorType, boolean alpha, + boolean progressive) { + this.format = format; + this.width = width; + this.height = height; + this.bitDepth = bitDepth; + this.colorType = colorType; + this.alpha = alpha; + this.progressive = progressive; + } + + /// Reads the header of an image held in memory; null when the data is not a + /// recognized format or its header is truncated or invalid. + public static ImageInfo read(byte[] data) { + if (data == null || data.length < 12) { + return null; + } + int b0 = data[0] & 0xff; + if (b0 == 0x89 && ascii(data, 1, "PNG")) { + return readPng(data); + } + if (b0 == 0xff && (data[1] & 0xff) == 0xd8) { + return readJpeg(data); + } + if (ascii(data, 0, "GIF87a") || ascii(data, 0, "GIF89a")) { + return new ImageInfo(Format.GIF, le16(data, 6), le16(data, 8), -1, -1, false, false); + } + if (ascii(data, 0, "RIFF") && ascii(data, 8, "WEBP")) { + return readWebp(data); + } + return null; + } + + /// Reads `in` to its end (at most 64 MB) and answers [#read(byte[])] of the + /// bytes. The stream is not closed. + /// + /// #### Throws + /// + /// - `IOException`: when reading fails or the stream is longer than 64 MB. + public static ImageInfo read(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) > 0) { + if (out.size() + n > MAX_STREAM_BYTES) { + throw new IOException("image larger than " + MAX_STREAM_BYTES + " bytes"); + } + out.write(buf, 0, n); + } + return read(out.toByteArray()); + } + + /// The format. + public Format getFormat() { + return format; + } + + /// The width in pixels. + public int getWidth() { + return width; + } + + /// The height in pixels. + public int getHeight() { + return height; + } + + /// Bits per sample: the PNG bit depth or the JPEG sample precision; -1 for + /// GIF and WebP. + public int getBitDepth() { + return bitDepth; + } + + /// The PNG color type (0 gray, 2 RGB, 3 palette, 4 gray + alpha, 6 RGBA); -1 + /// for the other formats. + public int getColorType() { + return colorType; + } + + /// True when the image can carry transparency: a PNG with an alpha color type + /// or a `tRNS` chunk, or a WebP with an alpha flag. Always false for JPEG and + /// GIF. + public boolean hasAlpha() { + return alpha; + } + + /// True for a progressive JPEG (`SOF2`, `SOF6`, `SOF10`, `SOF14`). + public boolean isProgressive() { + return progressive; + } + + @Override + public String toString() { + return format + " " + width + "x" + height; + } + + private static ImageInfo readPng(byte[] d) { + if (d.length < 29 || (d[4] & 0xff) != 0x0d || (d[5] & 0xff) != 0x0a || (d[6] & 0xff) != 0x1a + || (d[7] & 0xff) != 0x0a || !ascii(d, 12, "IHDR")) { + return null; + } + int w = be32(d, 16); + int h = be32(d, 20); + if (w <= 0 || h <= 0) { + return null; + } + int depth = d[24] & 0xff; + int type = d[25] & 0xff; + boolean alpha = type == 4 || type == 6; + // Walk the chunks after IHDR for a tRNS before the image data. + int p = 8 + 8 + 13 + 4; + while (!alpha && p + 8 <= d.length) { + int len = be32(d, p); + if (len < 0 || ascii(d, p + 4, "IDAT") || ascii(d, p + 4, "IEND")) { + break; + } + if (ascii(d, p + 4, "tRNS")) { + alpha = true; + } + long next = (long) p + 12 + len; + if (next > d.length) { + break; + } + p = (int) next; + } + return new ImageInfo(Format.PNG, w, h, depth, type, alpha, false); + } + + private static ImageInfo readJpeg(byte[] d) { + int p = 2; + int len = d.length; + while (p < len) { + if ((d[p] & 0xff) != 0xff) { + return null; + } + // Any number of 0xFF fill bytes may precede a marker. + while (p < len && (d[p] & 0xff) == 0xff) { + p++; + } + if (p >= len) { + return null; + } + int marker = d[p++] & 0xff; + if (marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { + continue; // TEM and RSTn stand alone. + } + if (marker == 0xd9 || marker == 0xda || marker == 0x00) { + return null; // End of image, or scan data, before any frame. + } + if (p + 2 > len) { + return null; + } + int segment = ((d[p] & 0xff) << 8) | (d[p + 1] & 0xff); + if (segment < 2) { + return null; + } + if (marker >= 0xc0 && marker <= 0xcf && marker != 0xc4 && marker != 0xc8 && marker != 0xcc) { + if (p + 7 > len) { + return null; + } + int precision = d[p + 2] & 0xff; + int h = ((d[p + 3] & 0xff) << 8) | (d[p + 4] & 0xff); + int w = ((d[p + 5] & 0xff) << 8) | (d[p + 6] & 0xff); + if (w == 0 || h == 0) { + return null; + } + boolean progressive = marker == 0xc2 || marker == 0xc6 || marker == 0xca || marker == 0xce; + return new ImageInfo(Format.JPEG, w, h, precision, -1, false, progressive); + } + p += segment; + } + return null; + } + + private static ImageInfo readWebp(byte[] d) { + if (d.length < 30) { + return null; + } + if (ascii(d, 12, "VP8 ")) { + if ((d[23] & 0xff) != 0x9d || (d[24] & 0xff) != 0x01 || (d[25] & 0xff) != 0x2a) { + return null; + } + return new ImageInfo(Format.WEBP, le16(d, 26) & 0x3fff, le16(d, 28) & 0x3fff, -1, -1, false, false); + } + if (ascii(d, 12, "VP8L")) { + if ((d[20] & 0xff) != 0x2f) { + return null; + } + int b1 = d[21] & 0xff; + int b2 = d[22] & 0xff; + int b3 = d[23] & 0xff; + int b4 = d[24] & 0xff; + int w = 1 + (b1 | ((b2 & 0x3f) << 8)); + int h = 1 + ((b2 >> 6) | (b3 << 2) | ((b4 & 0x0f) << 10)); + boolean alpha = ((b4 >> 4) & 1) != 0; + return new ImageInfo(Format.WEBP, w, h, -1, -1, alpha, false); + } + if (ascii(d, 12, "VP8X")) { + boolean alpha = (d[20] & 0x10) != 0; + int w = 1 + le24(d, 24); + int h = 1 + le24(d, 27); + return new ImageInfo(Format.WEBP, w, h, -1, -1, alpha, false); + } + return null; + } + + private static boolean ascii(byte[] d, int at, String s) { + if (at < 0 || at + s.length() > d.length) { + return false; + } + for (int i = 0; i < s.length(); i++) { + if (d[at + i] != s.charAt(i)) { + return false; + } + } + return true; + } + + private static int be32(byte[] d, int at) { + return ((d[at] & 0xff) << 24) | ((d[at + 1] & 0xff) << 16) | ((d[at + 2] & 0xff) << 8) | (d[at + 3] & 0xff); + } + + private static int le16(byte[] d, int at) { + return (d[at] & 0xff) | ((d[at + 1] & 0xff) << 8); + } + + private static int le24(byte[] d, int at) { + return (d[at] & 0xff) | ((d[at + 1] & 0xff) << 8) | ((d[at + 2] & 0xff) << 16); + } +} diff --git a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java new file mode 100644 index 00000000000..39bdf940a96 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.xml; + +import com.codename1.backend.Base64; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// Reads Apple XML property lists: provisioning profiles, entitlements, +/// `Info.plist` files. +/// +/// ```java +/// Map profile = (Map) PropertyLists.parse(mobileprovisionBytes); +/// String team = (String) ((List) profile.get("TeamIdentifier")).get(0); +/// Date expires = (Date) profile.get("ExpirationDate"); +/// ``` +/// +/// Values map to Java types as follows: +/// +/// | plist | Java | +/// |---|---| +/// | `` | `Map`, in document order | +/// | `` | `List` | +/// | `` | `String` | +/// | `` | `Long` | +/// | `` | `Double` | +/// | `` `` | `Boolean` | +/// | `` | `byte[]`, base64 decoded | +/// | `` | `java.util.Date` (UTC, as `2025-01-31T12:00:00Z` is written) | +/// +/// Check a value's type with `instanceof` before casting it. +/// +/// A provisioning profile (`.mobileprovision`) is not a plist itself but a +/// CMS/PKCS#7 signed envelope with the XML plist inside it. [#parse(byte[])] +/// accepts it as it is, by locating the plist with [#extractXml(byte[])]; the +/// signature is not verified. Binary property lists (`bplist00`) are not +/// supported and are reported as such. +/// +/// Parsing goes through [XmlPullParser], so a document type declaration is never +/// interpreted and nesting is bounded: a hostile plist cannot fetch a URL, +/// expand entities or exhaust the stack. +public final class PropertyLists { + private PropertyLists() { + } + + /// Parses a property list -- an XML plist, or any binary blob that wraps one, + /// such as a signed provisioning profile -- and answers its top-level value, + /// usually a `Map`. + /// + /// #### Throws + /// + /// - `XmlException`: when no XML plist is found, the data is a binary plist, + /// or the plist is malformed (an unknown element, a `` key with no + /// value, an integer that does not parse). + public static Object parse(byte[] data) throws XmlException { + if (data == null) { + throw new XmlException("no data"); + } + if (startsWith(data, 0, "bplist")) { + throw new XmlException("binary property lists are not supported"); + } + int first = 0; + while (first < data.length && isSpace(data[first])) { + first++; + } + boolean bom = startsWithBom(data, first); + XmlElement root; + if (bom || (first < data.length && data[first] == '<')) { + root = XmlElement.parse(data); + } else { + byte[] xml = extractXml(data); + if (xml == null) { + throw new XmlException("no XML property list found"); + } + root = XmlElement.parse(xml); + } + return fromDocument(root); + } + + /// Parses an XML property list held in a string. + /// + /// #### Throws + /// + /// - `XmlException`: when the plist is malformed. + public static Object parse(String xml) throws XmlException { + return fromDocument(XmlElement.parse(xml)); + } + + /// Finds the XML property list embedded in binary data and answers its bytes, + /// from `` after it; null when there is none. + /// + /// This is how the plist is read out of a provisioning profile, which stores + /// it unencrypted inside a signed CMS envelope. The bytes before and after the + /// span are not examined. + public static byte[] extractXml(byte[] data) { + if (data == null) { + return null; + } + int start = indexOf(data, "= 0 && plist < start)) { + start = plist; + } + if (start < 0) { + return null; + } + int end = indexOf(data, "", start); + if (end < 0) { + return null; + } + end += "".length(); + byte[] out = new byte[end - start]; + System.arraycopy(data, start, out, 0, out.length); + return out; + } + + private static Object fromDocument(XmlElement root) throws XmlException { + if (!"plist".equals(root.getName())) { + return value(root); + } + List kids = root.getChildren(); + if (kids.size() != 1) { + throw new XmlException(" must hold exactly one value, found " + kids.size()); + } + return value(kids.get(0)); + } + + private static Object value(XmlElement e) throws XmlException { + String n = e.getName(); + if ("dict".equals(n)) { + Map map = new LinkedHashMap(); + List kids = e.getChildren(); + int count = kids.size(); + for (int i = 0; i < count; i += 2) { + XmlElement key = kids.get(i); + if (!"key".equals(key.getName())) { + throw new XmlException("expected in , found <" + key.getName() + ">"); + } + if (i + 1 >= count) { + throw new XmlException("" + key.getText() + " has no value"); + } + map.put(key.getText(), value(kids.get(i + 1))); + } + return map; + } + if ("array".equals(n)) { + List list = new ArrayList(); + for (XmlElement c : e.getChildren()) { + list.add(value(c)); + } + return list; + } + if ("string".equals(n)) { + return e.getText(); + } + if ("integer".equals(n)) { + String t = e.getText().trim(); + try { + return Long.valueOf(Long.parseLong(t)); + } catch (NumberFormatException ex) { + throw new XmlException("bad " + t, ex); + } + } + if ("real".equals(n)) { + String t = e.getText().trim(); + try { + return Double.valueOf(Double.parseDouble(t)); + } catch (NumberFormatException ex) { + throw new XmlException("bad " + t, ex); + } + } + if ("true".equals(n)) { + return Boolean.TRUE; + } + if ("false".equals(n)) { + return Boolean.FALSE; + } + if ("data".equals(n)) { + byte[] bytes = Base64.decode(stripWhitespace(e.getText())); + if (bytes == null) { + throw new XmlException("bad base64 in "); + } + return bytes; + } + if ("date".equals(n)) { + return parseDate(e.getText().trim()); + } + throw new XmlException("unknown property list element <" + n + ">"); + } + + /// Parses the one form plists write, `yyyy-MM-ddTHH:mm:ssZ`, as UTC. + private static Date parseDate(String s) throws XmlException { + if (s.length() != 20 || s.charAt(4) != '-' || s.charAt(7) != '-' || s.charAt(10) != 'T' + || s.charAt(13) != ':' || s.charAt(16) != ':' || s.charAt(19) != 'Z') { + throw new XmlException("bad " + s); + } + int year = digits(s, 0, 4); + int month = digits(s, 5, 2); + int day = digits(s, 8, 2); + int hour = digits(s, 11, 2); + int minute = digits(s, 14, 2); + int second = digits(s, 17, 2); + if (year < 0 || month < 1 || month > 12 || day < 1 || day > 31 + || hour < 0 || hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 60) { + throw new XmlException("bad " + s); + } + // Days from the civil calendar (H. Hinnant's algorithm), proleptic Gregorian. + int y = month <= 2 ? year - 1 : year; + int era = y / 400; + int yoe = y - era * 400; + int mp = (month + 9) % 12; + int doy = (153 * mp + 2) / 5 + day - 1; + int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + long days = (long) era * 146097 + doe - 719468; + long seconds = days * 86400L + hour * 3600L + minute * 60L + second; + return new Date(seconds * 1000L); + } + + private static int digits(String s, int start, int count) { + int v = 0; + for (int i = start; i < start + count; i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + return -1; + } + v = v * 10 + (c - '0'); + } + return v; + } + + private static String stripWhitespace(String s) { + StringBuilder b = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + b.append(c); + } + } + return b.toString(); + } + + private static boolean isSpace(byte b) { + return b == ' ' || b == '\t' || b == '\r' || b == '\n'; + } + + private static boolean startsWithBom(byte[] data, int at) { + return at + 2 < data.length && (data[at] & 0xff) == 0xef && (data[at + 1] & 0xff) == 0xbb + && (data[at + 2] & 0xff) == 0xbf; + } + + private static boolean startsWith(byte[] data, int at, String ascii) { + if (at + ascii.length() > data.length) { + return false; + } + for (int i = 0; i < ascii.length(); i++) { + if (data[at + i] != ascii.charAt(i)) { + return false; + } + } + return true; + } + + private static int indexOf(byte[] data, String ascii, int from) { + int last = data.length - ascii.length(); + for (int i = from; i <= last; i++) { + if (startsWith(data, i, ascii)) { + return i; + } + } + return -1; + } +} diff --git a/vm/backend/src/com/codename1/backend/xml/XmlElement.java b/vm/backend/src/com/codename1/backend/xml/XmlElement.java new file mode 100644 index 00000000000..31fc81bc738 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/xml/XmlElement.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.xml; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// One element of a small in-memory XML tree, built by [XmlPullParser]. +/// +/// ```java +/// XmlElement root = XmlElement.parse(bytes); +/// for (XmlElement item : root.children("item")) { +/// String id = item.getAttribute("id"); +/// String title = item.child("title").getText(); +/// } +/// ``` +/// +/// The tree keeps elements, attributes and text; comments, processing +/// instructions and the document type are dropped, exactly as the parser drops +/// them. It carries every limit of the parser, including its refusal to expand +/// any entity a document type declares, so it is safe for untrusted input. +public final class XmlElement { + private final String name; + private final Map attributes; + private final List children = new ArrayList(); + private String text = ""; + + private XmlElement(String name, Map attributes) { + this.name = name; + this.attributes = attributes; + } + + /// Parses a UTF-8 document and answers its root element. + /// + /// #### Throws + /// + /// - `XmlException`: when the document is not well formed; see [XmlPullParser]. + public static XmlElement parse(byte[] utf8) throws XmlException { + return parse(XmlPullParser.of(utf8)); + } + + /// Parses a document held in a string and answers its root element. + /// + /// #### Throws + /// + /// - `XmlException`: when the document is not well formed; see [XmlPullParser]. + public static XmlElement parse(String document) throws XmlException { + return parse(XmlPullParser.of(document)); + } + + /// Reads `in` to its end as UTF-8 and answers the root element. The stream is + /// not closed. + /// + /// #### Throws + /// + /// - `IOException`: when reading fails, or an `XmlException` when the + /// document is not well formed. + public static XmlElement parse(InputStream in) throws IOException { + return parse(XmlPullParser.of(in)); + } + + /// Builds the tree from a parser positioned before the root element, reading + /// it to the end of the document. Configure the parser first, for example + /// with [XmlPullParser#setMaxDepth(int)]. + /// + /// #### Throws + /// + /// - `XmlException`: when the document is not well formed. + public static XmlElement parse(XmlPullParser parser) throws XmlException { + List open = new ArrayList(); + XmlElement root = null; + for (int ev = parser.next(); ev != XmlPullParser.END_DOCUMENT; ev = parser.next()) { + if (ev == XmlPullParser.START_ELEMENT) { + int count = parser.getAttributeCount(); + Map attrs = count == 0 + ? Collections.emptyMap() + : new LinkedHashMap(); + for (int i = 0; i < count; i++) { + attrs.put(parser.getAttributeName(i), parser.getAttributeValue(i)); + } + XmlElement e = new XmlElement(parser.getName(), attrs); + if (open.isEmpty()) { + root = e; + } else { + open.get(open.size() - 1).children.add(e); + } + open.add(e); + } else if (ev == XmlPullParser.END_ELEMENT) { + open.remove(open.size() - 1); + } else if (ev == XmlPullParser.TEXT && !open.isEmpty()) { + XmlElement e = open.get(open.size() - 1); + e.text = e.text.concat(parser.getText()); + } + } + if (root == null) { + throw new XmlException("no root element"); + } + return root; + } + + /// The element name as written, prefix included. + public String getName() { + return name; + } + + /// The decoded value of the named attribute, or null when there is none. + public String getAttribute(String attributeName) { + return attributes.get(attributeName); + } + + /// Every attribute, in document order. The map is read-only. + public Map getAttributes() { + return Collections.unmodifiableMap(attributes); + } + + /// The child elements in document order. The list is read-only. + public List getChildren() { + return Collections.unmodifiableList(children); + } + + /// The first child element named `childName`, or null when there is none. + public XmlElement child(String childName) { + for (XmlElement c : children) { + if (c.name.equals(childName)) { + return c; + } + } + return null; + } + + /// Every child element named `childName`, in document order; empty when there + /// is none. + public List children(String childName) { + List out = new ArrayList(); + for (XmlElement c : children) { + if (c.name.equals(childName)) { + out.add(c); + } + } + return out; + } + + /// The element's own character data -- every text and CDATA run directly + /// inside it, concatenated, with whitespace as written. Text inside child + /// elements is not included. + public String getText() { + return text; + } + + /// The text of the first child named `childName`, trimmed, or null when there + /// is no such child. + public String childText(String childName) { + XmlElement c = child(childName); + return c == null ? null : c.getText().trim(); + } + + @Override + public String toString() { + return "<" + name + "> with " + children.size() + " children"; + } +} diff --git a/vm/backend/src/com/codename1/backend/xml/XmlException.java b/vm/backend/src/com/codename1/backend/xml/XmlException.java new file mode 100644 index 00000000000..00d60339cdf --- /dev/null +++ b/vm/backend/src/com/codename1/backend/xml/XmlException.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.xml; + +import java.io.IOException; + +/// Thrown when a document is not well-formed XML, uses a construct this parser +/// refuses (an entity it does not know, nesting past the limit), or is not the +/// structure a reader such as [PropertyLists] expects. +/// +/// The message names the problem and, where one is known, the character offset +/// in the decoded document. +public class XmlException extends IOException { + private static final long serialVersionUID = 1L; + + /// An exception with the given message. + public XmlException(String message) { + super(message); + } + + /// An exception with the given message, caused by `cause`. + public XmlException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/vm/backend/src/com/codename1/backend/xml/XmlPullParser.java b/vm/backend/src/com/codename1/backend/xml/XmlPullParser.java new file mode 100644 index 00000000000..b908ff21012 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/xml/XmlPullParser.java @@ -0,0 +1,516 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.xml; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +/// A small streaming XML parser: call [#next()] until it answers +/// [#END_DOCUMENT], reading the element name, attributes and text of each event +/// as it goes. +/// +/// ```java +/// XmlPullParser p = XmlPullParser.of(bytes); +/// for (int ev = p.next(); ev != XmlPullParser.END_DOCUMENT; ev = p.next()) { +/// if (ev == XmlPullParser.START_ELEMENT && "item".equals(p.getName())) { +/// String id = p.getAttributeValue("id"); +/// } +/// } +/// ``` +/// +/// What it does: +/// +/// - Elements, attributes (single or double quoted) and text. An empty element +/// (``) is reported as a start followed immediately by its end. +/// - Text and CDATA sections are reported together as one [#TEXT] event per run +/// of character data between two tags; whitespace is kept as written. Text +/// outside the root element is ignored. +/// - The five predefined entities (`&` `<` `>` `"` `'`) and +/// numeric character references (`A`, `A`) are decoded, in text and in +/// attribute values. +/// - Namespaces are not resolved: a name is reported as written, prefix included +/// (`plist:dict`, `xmlns:a`). +/// - Comments, processing instructions and the XML declaration are skipped. +/// +/// What it refuses, by construction rather than by configuration: +/// +/// - **A document type declaration is skipped, never interpreted.** Its internal +/// subset is not read, so an entity it declares is never defined, an external +/// one is never fetched, and a reference to either is an [XmlException]. That +/// makes the parser immune to external entity (XXE) and entity expansion +/// ("billion laughs") attacks: there is nothing to expand. +/// - Elements nested deeper than [#setMaxDepth(int)] (256 by default) are +/// rejected, so a hostile document cannot make a recursive consumer such as +/// [XmlElement] overflow its stack. +/// - A document that is not well formed -- a mismatched end tag, an unclosed +/// element, a second root element, a stray `<` -- is an [XmlException]. +/// +/// Input is decoded as UTF-8; a leading byte order mark is skipped. The +/// `encoding` of the XML declaration is not consulted. +public final class XmlPullParser { + /// An element's start tag; [#getName()] and the attribute accessors describe it. + public static final int START_ELEMENT = 1; + /// An element's end tag; [#getName()] names the element being closed. + public static final int END_ELEMENT = 2; + /// Character data, with entities decoded; read it with [#getText()]. + public static final int TEXT = 3; + /// The end of the document. Every later call to [#next()] answers this again. + public static final int END_DOCUMENT = 4; + + private static final int DEFAULT_MAX_DEPTH = 256; + + private final String in; + private int pos; + private int maxDepth = DEFAULT_MAX_DEPTH; + private final List stack = new ArrayList(); + private boolean rootSeen; + private boolean pendingEnd; + private boolean done; + + private int event; + private String name; + private String text; + private final List attrNames = new ArrayList(); + private final List attrValues = new ArrayList(); + + private XmlPullParser(String document) { + this.in = document; + if (document.length() > 0 && document.charAt(0) == (char) 0xfeff) { + pos = 1; + } + } + + /// A parser over a document that is already a string. + public static XmlPullParser of(String document) { + if (document == null) { + throw new IllegalArgumentException("document is null"); + } + return new XmlPullParser(document); + } + + /// A parser over UTF-8 encoded bytes. + public static XmlPullParser of(byte[] utf8) { + return of(utf8, 0, utf8.length); + } + + /// A parser over `length` UTF-8 encoded bytes of `data` starting at `offset`. + public static XmlPullParser of(byte[] data, int offset, int length) { + try { + return new XmlPullParser(new String(data, offset, length, "UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("UTF-8 is not supported", e); + } + } + + /// A parser over the whole of `in`, read as UTF-8. The stream is read to its + /// end but not closed. + public static XmlPullParser of(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) > 0) { + out.write(buf, 0, n); + } + return of(out.toByteArray()); + } + + /// Sets the deepest element nesting accepted; deeper is an [XmlException]. + /// Answers this parser. + public XmlPullParser setMaxDepth(int maxDepth) { + if (maxDepth < 1) { + throw new IllegalArgumentException("maxDepth must be positive"); + } + this.maxDepth = maxDepth; + return this; + } + + /// The current event: one of [#START_ELEMENT], [#END_ELEMENT], [#TEXT] or + /// [#END_DOCUMENT], or 0 before the first call to [#next()]. + public int getEventType() { + return event; + } + + /// The element name of the current start or end event, prefix included, or + /// null for any other event. + public String getName() { + return name; + } + + /// The decoded character data of the current [#TEXT] event, or null for any + /// other event. + public String getText() { + return text; + } + + /// The number of elements currently open: 1 inside the root element, 0 before + /// and after it. After an [#END_ELEMENT] event the closed element no longer + /// counts. + public int getDepth() { + return stack.size(); + } + + /// The number of attributes on the current start element; 0 for any other event. + public int getAttributeCount() { + return attrNames.size(); + } + + /// The name of attribute `index` of the current start element, as written. + public String getAttributeName(int index) { + return attrNames.get(index); + } + + /// The decoded value of attribute `index` of the current start element. + public String getAttributeValue(int index) { + return attrValues.get(index); + } + + /// The decoded value of the attribute named `attributeName` (prefix included) + /// on the current start element, or null when it has none. + public String getAttributeValue(String attributeName) { + int count = attrNames.size(); + for (int i = 0; i < count; i++) { + if (attrNames.get(i).equals(attributeName)) { + return attrValues.get(i); + } + } + return null; + } + + /// Advances to the next event and answers its type. + /// + /// #### Throws + /// + /// - `XmlException`: when the document is not well formed, nests too deeply, + /// or references an entity this parser does not define. + public int next() throws XmlException { + name = null; + text = null; + attrNames.clear(); + attrValues.clear(); + if (pendingEnd) { + pendingEnd = false; + name = stack.remove(stack.size() - 1); + return event = END_ELEMENT; + } + if (done) { + return event = END_DOCUMENT; + } + StringBuilder chars = null; + int len = in.length(); + boolean atTag = false; + while (!atTag) { + if (pos >= len) { + if (!stack.isEmpty()) { + throw error("unexpected end of document inside <" + stack.get(stack.size() - 1) + ">"); + } + if (!rootSeen) { + throw error("no root element"); + } + if (chars != null && !isWhitespace(chars)) { + throw error("text outside the root element"); + } + done = true; + return event = END_DOCUMENT; + } + char c = in.charAt(pos); + if (c != '<') { + if (chars == null) { + chars = new StringBuilder(); + } + if (c == '&') { + readReference(chars); + } else { + chars.append(c); + pos++; + } + continue; + } + if (in.startsWith("", pos + 9); + if (end < 0) { + throw error("unterminated CDATA section"); + } + if (chars == null) { + chars = new StringBuilder(); + } + chars.append(in, pos + 9, end); + pos = end + 3; + continue; + } + if (in.startsWith("", "comment"); + continue; + } + if (in.startsWith("", "processing instruction"); + continue; + } + if (in.startsWith(""); + } + while (true) { + skipWhitespace(); + if (pos >= in.length()) { + throw error("unterminated start tag <" + tag + ">"); + } + char c = in.charAt(pos); + if (c == '>') { + pos++; + break; + } + if (c == '/') { + if (pos + 1 >= in.length() || in.charAt(pos + 1) != '>') { + throw error("expected /> in <" + tag + ">"); + } + pos += 2; + pendingEnd = true; + break; + } + String attr = readName(); + skipWhitespace(); + expect('='); + skipWhitespace(); + if (pos >= in.length()) { + throw error("unterminated attribute " + attr); + } + char quote = in.charAt(pos); + if (quote != '"' && quote != '\'') { + throw error("attribute " + attr + " is not quoted"); + } + pos++; + StringBuilder value = new StringBuilder(); + while (true) { + if (pos >= in.length()) { + throw error("unterminated value of attribute " + attr); + } + char v = in.charAt(pos); + if (v == quote) { + pos++; + break; + } + if (v == '<') { + throw error("< in the value of attribute " + attr); + } + if (v == '&') { + readReference(value); + } else { + value.append(v); + pos++; + } + } + if (attrNames.contains(attr)) { + throw error("duplicate attribute " + attr); + } + attrNames.add(attr); + attrValues.add(value.toString()); + } + if (stack.size() >= maxDepth) { + throw error("elements nested deeper than " + maxDepth); + } + stack.add(tag); + rootSeen = true; + name = tag; + return event = START_ELEMENT; + } + + private int readEndTag() throws XmlException { + pos += 2; + String tag = readName(); + skipWhitespace(); + expect('>'); + if (stack.isEmpty()) { + throw error("end tag with no open element"); + } + String open = stack.remove(stack.size() - 1); + if (!open.equals(tag)) { + throw error("end tag does not match <" + open + ">"); + } + name = tag; + return event = END_ELEMENT; + } + + private void readReference(StringBuilder out) throws XmlException { + int semi = in.indexOf(';', pos); + if (semi < 0 || semi - pos > 12) { + throw error("unterminated entity reference"); + } + String ref = in.substring(pos + 1, semi); + pos = semi + 1; + if ("amp".equals(ref)) { + out.append('&'); + } else if ("lt".equals(ref)) { + out.append('<'); + } else if ("gt".equals(ref)) { + out.append('>'); + } else if ("quot".equals(ref)) { + out.append('"'); + } else if ("apos".equals(ref)) { + out.append('\''); + } else if (ref.length() > 1 && ref.charAt(0) == '#') { + int code = parseCharRef(ref); + if (code < 0 || code > 0x10ffff || code == 0 || (code >= 0xd800 && code <= 0xdfff)) { + throw error("invalid character reference &" + ref + ";"); + } + if (code >= 0x10000) { + int v = code - 0x10000; + out.append((char) (0xd800 + (v >> 10))); + out.append((char) (0xdc00 + (v & 0x3ff))); + } else { + out.append((char) code); + } + } else { + // Deliberately no DTD-declared entities: see the class comment. + throw error("undefined entity &" + ref + ";"); + } + } + + private static int parseCharRef(String ref) { + boolean hex = ref.length() > 2 && (ref.charAt(1) == 'x' || ref.charAt(1) == 'X'); + int start = hex ? 2 : 1; + if (start >= ref.length()) { + return -1; + } + int value = 0; + for (int i = start; i < ref.length(); i++) { + int digit = Character.digit(ref.charAt(i), hex ? 16 : 10); + if (digit < 0) { + return -1; + } + value = value * (hex ? 16 : 10) + digit; + if (value > 0x10ffff) { + return -1; + } + } + return value; + } + + /// Skips `` (or any other `' && brackets <= 0) { + return; + } + } + throw error("unterminated declaration"); + } + + private void skipPast(String terminator, String what) throws XmlException { + int end = in.indexOf(terminator, pos + 2); + if (end < 0) { + throw error("unterminated " + what); + } + pos = end + terminator.length(); + } + + private String readName() throws XmlException { + int start = pos; + int len = in.length(); + while (pos < len) { + char c = in.charAt(pos); + if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '>' || c == '/' + || c == '=' || c == '<' || c == '"' || c == '\'') { + break; + } + pos++; + } + if (pos == start) { + throw error("expected a name"); + } + return in.substring(start, pos); + } + + private void expect(char c) throws XmlException { + if (pos >= in.length() || in.charAt(pos) != c) { + throw error("expected '" + c + "'"); + } + pos++; + } + + private void skipWhitespace() { + int len = in.length(); + while (pos < len && isSpace(in.charAt(pos))) { + pos++; + } + } + + private static boolean isSpace(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; + } + + private static boolean isWhitespace(CharSequence s) { + for (int i = 0; i < s.length(); i++) { + if (!isSpace(s.charAt(i))) { + return false; + } + } + return true; + } + + private XmlException error(String message) { + return new XmlException(message + " at offset " + pos); + } +} diff --git a/vm/backend/src/com/codename1/backend/xml/package-info.java b/vm/backend/src/com/codename1/backend/xml/package-info.java new file mode 100644 index 00000000000..5070443c9de --- /dev/null +++ b/vm/backend/src/com/codename1/backend/xml/package-info.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// Reading XML on the server: a streaming pull parser, a small element tree and +/// an Apple property list reader. +/// +/// [XmlPullParser] is the parser everything else is built on; [XmlElement] is a +/// tree for documents small enough to hold in memory; [PropertyLists] reads XML +/// property lists, including the one inside a signed provisioning profile. +/// +/// All three are safe for untrusted input by construction. A document type +/// declaration is skipped and never interpreted, so no external entity is ever +/// resolved and no entity is ever expanded beyond the five XML predefines and +/// numeric character references; nesting depth is capped. There is no switch +/// that turns either protection off. +package com.codename1.backend.xml; From 8fc7c9e5b47a8ca51c0432d87544dda4d9a83ce6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:13:08 +0200 Subject: [PATCH 07/25] Backend: HTTP/2 on the JVM WebClient through java.net.http Co-Authored-By: Claude Opus 5.5 --- .../codename1/backend/WebClientHttp2Test.java | 65 +++++ .../codename1/backend/ClientCredentials.java | 19 +- .../com/codename1/backend/JdkHttp2.java | 272 ++++++++++++++++++ .../javase/com/codename1/backend/Web.java | 23 +- .../src/com/codename1/backend/WebClient.java | 9 +- 5 files changed, 371 insertions(+), 17 deletions(-) create mode 100644 maven/backend/src/test/java/com/codename1/backend/WebClientHttp2Test.java create mode 100644 vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java diff --git a/maven/backend/src/test/java/com/codename1/backend/WebClientHttp2Test.java b/maven/backend/src/test/java/com/codename1/backend/WebClientHttp2Test.java new file mode 100644 index 00000000000..0ec6245b08e --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/WebClientHttp2Test.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; + +import org.junit.jupiter.api.Test; + +/// A WebClient that requires HTTP/2 goes through the JDK's java.net.http client +/// on this arm, and a server that answers in HTTP/1.1 fails the request instead +/// of serving it over the protocol the caller ruled out. +class WebClientHttp2Test { + @Test + void anHttp11ServerIsRefusedWhenHttp2IsRequired() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + byte[] body = "one-one".getBytes("UTF-8"); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + server.start(); + WebClient client = new WebClient().setRequireHttp2(true); + try { + client.get("http://127.0.0.1:" + server.getAddress().getPort() + "/"); + fail("an HTTP/1.1 answer must not satisfy a client that requires HTTP/2"); + } catch (IOException err) { + String message = String.valueOf(err.getMessage()); + boolean java8 = System.getProperty("java.version").startsWith("1."); + assertTrue(java8 ? message.indexOf("Java 11 or newer") >= 0 + : message.indexOf("did not negotiate HTTP/2") >= 0, message); + } finally { + client.close(); + server.stop(0); + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/ClientCredentials.java b/vm/backend/impl/javase/com/codename1/backend/ClientCredentials.java index fed8669ca18..631fdd63454 100644 --- a/vm/backend/impl/javase/com/codename1/backend/ClientCredentials.java +++ b/vm/backend/impl/javase/com/codename1/backend/ClientCredentials.java @@ -57,6 +57,23 @@ private ClientCredentials() { } static SSLSocketFactory socketFactory(ClientCertificate credential) throws IOException { + return context(credential).getSocketFactory(); + } + + /// The TLS context presenting `credential`, or default trust and no key + /// for null: what the HTTP/2 client is built over, as the factories above + /// are for HttpURLConnection. + static SSLContext context(ClientCertificate credential) throws IOException { + if (credential == null) { + try { + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, null, null); + return context; + } catch (GeneralSecurityException err) { + throw new IOException("TLS is not available: " + err.getClass().getSimpleName(), + err); + } + } try { KeyStore store; char[] password; @@ -102,7 +119,7 @@ static SSLSocketFactory socketFactory(ClientCertificate credential) throws IOExc keys.init(store, password); SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), null, null); - return context.getSocketFactory(); + return context; } catch (GeneralSecurityException err) { throw new IOException("The client certificate could not be used: " + err.getClass().getSimpleName(), err); diff --git a/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java new file mode 100644 index 00000000000..a8f06669e3c --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URI; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/// HTTP/2 for the Java SE runtime, through the JDK's own `java.net.http` +/// client: what a `WebClient` that requires HTTP/2 uses on the JVM arm, so APNs +/// and anything else that speaks only HTTP/2 can be exercised in the +/// development loop and in tests. +/// +/// Reached by reflection because this arm compiles for Java 8 and +/// `java.net.http` arrived in 11. The classes are resolved through the system +/// class loader, never `Class.forName` (see BanClassForName in the pom), and on +/// a JVM without them the request fails saying so. +/// +/// The rules HttpURLConnection follows on the other path hold here too: a +/// redirect is followed only for a safe method with no headers, body or client +/// certificate; the response is bounded by `CN1_WEB_MAX_RESPONSE_MB`; header +/// names come back lower case. The client has no read timeout, so a request +/// with no request timeout set is bounded by its read timeout as a whole. +final class JdkHttp2 { + private static final String P = "java.net.http."; + + private JdkHttp2() { + } + + /// One `HttpClient`, per redirect policy, for one WebClient: the + /// connection it keeps is the pool. + static final class Clients { + private final Object[] clients = new Object[2]; + + synchronized Object get(WebClient client, boolean follow) throws IOException { + int at = follow ? 1 : 0; + if (clients[at] == null) { + clients[at] = build(client, follow); + } + return clients[at]; + } + + synchronized void close() { + for (int iter = 0 ; iter < clients.length ; iter++) { + if (clients[iter] != null) { + // close() exists from JDK 21; before it the client's + // connections idle out on their own. + try { + Method m = clients[iter].getClass().getMethod("shutdownNow"); + m.setAccessible(true); + m.invoke(clients[iter]); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // Not on this JDK. + } + clients[iter] = null; + } + } + } + } + + private static Class type(String name) throws IOException { + try { + return ClassLoader.getSystemClassLoader().loadClass(P + name); + } catch (ClassNotFoundException err) { + throw new IOException("An HTTP/2 request on the local Java SE runtime needs " + + "Java 11 or newer, whose java.net.http client makes it; this JVM is " + + System.getProperty("java.version") + ".", err); + } + } + + private static Object call(Object target, Class owner, String name, Class[] params, + Object... args) throws IOException { + try { + return owner.getMethod(name, params).invoke(target, args); + } catch (InvocationTargetException err) { + Throwable cause = err.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } + if (cause instanceof InterruptedException) { + Thread.currentThread().interrupt(); + throw new IOException("The HTTP/2 request was interrupted", cause); + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new IOException(String.valueOf(cause), cause); + } catch (ReflectiveOperationException err) { + throw new IOException("java.net.http has no " + owner.getName() + "." + name, err); + } + } + + private static Object constant(String enumType, String name) throws IOException { + Class c = type(enumType); + for (Object value : c.getEnumConstants()) { + if (((Enum) value).name().equals(name)) { + return value; + } + } + throw new IOException("java.net.http has no " + enumType + "." + name); + } + + private static Object duration(long millis) throws IOException { + try { + return java.time.Duration.ofMillis(millis); + } catch (RuntimeException err) { + throw new IOException(err.toString(), err); + } + } + + private static Object build(WebClient client, boolean follow) throws IOException { + Class clientType = type("HttpClient"); + Class builderType = type("HttpClient$Builder"); + Object builder = call(null, clientType, "newBuilder", new Class[0]); + call(builder, builderType, "version", new Class[] {type("HttpClient$Version")}, + constant("HttpClient$Version", "HTTP_2")); + call(builder, builderType, "followRedirects", new Class[] {type("HttpClient$Redirect")}, + constant("HttpClient$Redirect", follow ? "NORMAL" : "NEVER")); + if (client.getConnectTimeoutMillis() > 0) { + call(builder, builderType, "connectTimeout", new Class[] {java.time.Duration.class}, + duration(client.getConnectTimeoutMillis())); + } + call(builder, builderType, "sslContext", new Class[] {javax.net.ssl.SSLContext.class}, + ClientCredentials.context(client.getClientCertificate())); + return call(builder, builderType, "build", new Class[0]); + } + + static Web.Result perform(WebClient client, Clients clients, String method, String url, + List headers, byte[] body, List trace, long maxBytes) + throws IOException { + String verb = method == null ? "GET" : method; + boolean safe = "GET".equals(verb) || "HEAD".equals(verb); + boolean follow = (headers == null || headers.isEmpty()) + && (body == null || body.length == 0) && safe + && client.getClientCertificate() == null; + Object http = clients.get(client, follow); + + Class requestType = type("HttpRequest"); + Class rbType = type("HttpRequest$Builder"); + Class publisherType = type("HttpRequest$BodyPublisher"); + Class publishers = type("HttpRequest$BodyPublishers"); + Object rb; + try { + rb = call(null, requestType, "newBuilder", new Class[] {URI.class}, new URI(url)); + } catch (java.net.URISyntaxException err) { + throw new IOException("Not a URL: " + Urls.forMessage(url), err); + } + Object publisher = body == null || body.length == 0 + ? call(null, publishers, "noBody", new Class[0]) + : call(null, publishers, "ofByteArray", new Class[] {byte[].class}, body); + call(rb, rbType, "method", new Class[] {String.class, publisherType}, verb, publisher); + boolean agent = false; + List lines = new ArrayList(); + if (headers != null) { + lines.addAll(headers); + } + if (trace != null) { + lines.addAll(trace); + } + for (Object line : lines) { + String header = String.valueOf(line); + int colon = header.indexOf(':'); + if (colon <= 0) { + continue; + } + String name = header.substring(0, colon).trim(); + String value = header.substring(colon + 1).trim(); + if ("Connection".equalsIgnoreCase(name) || "Content-Length".equalsIgnoreCase(name) + || "Host".equalsIgnoreCase(name)) { + // Connection-specific, and forbidden in HTTP/2 (RFC 9113 8.2.2); + // the client derives the other two. + continue; + } + agent |= "User-Agent".equalsIgnoreCase(name); + call(rb, rbType, "header", new Class[] {String.class, String.class}, name, value); + } + if (!agent) { + call(rb, rbType, "header", new Class[] {String.class, String.class}, + "User-Agent", "codenameone-backend"); + } + int ceiling = client.getRequestTimeoutMillis() > 0 ? client.getRequestTimeoutMillis() + : client.getReadTimeoutMillis(); + if (ceiling > 0) { + call(rb, rbType, "timeout", new Class[] {java.time.Duration.class}, + duration(ceiling)); + } + Object request = call(rb, rbType, "build", new Class[0]); + + Class handlers = type("HttpResponse$BodyHandlers"); + Class handlerType = type("HttpResponse$BodyHandler"); + Object handler = call(null, handlers, "ofInputStream", new Class[0]); + Class responseType = type("HttpResponse"); + Object response = call(http, type("HttpClient"), "send", + new Class[] {requestType, handlerType}, request, handler); + InputStream in = (InputStream) call(response, responseType, "body", new Class[0]); + try { + Object version = call(response, responseType, "version", new Class[0]); + if (!"HTTP_2".equals(((Enum) version).name())) { + throw new IOException("Request to " + Urls.forMessage(url) + " failed: the " + + "server did not negotiate HTTP/2, which this client requires"); + } + int status = ((Integer) call(response, responseType, "statusCode", + new Class[0])).intValue(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + long total = 0; + int n; + while ((n = in.read(chunk)) > 0) { + total += n; + if (total > maxBytes) { + throw new IOException("Request to " + Urls.forMessage(url) + " failed: the " + + "response is larger than the " + (maxBytes / (1024 * 1024)) + + "MB CN1_WEB_MAX_RESPONSE_MB allows"); + } + buffer.write(chunk, 0, n); + } + Object httpHeaders = call(response, responseType, "headers", new Class[0]); + Map all = (Map) call(httpHeaders, type("HttpHeaders"), "map", new Class[0]); + Map values = new LinkedHashMap(); + for (Iterator it = all.entrySet().iterator(); it.hasNext();) { + Map.Entry e = (Map.Entry) it.next(); + String name = String.valueOf(e.getKey()); + if (name.startsWith(":")) { + continue; // a pseudo-header, not a field + } + values.put(asciiLower(name), new ArrayList((List) e.getValue())); + } + return new Web.Result(status, buffer.toByteArray(), null, values, "HTTP/2"); + } finally { + in.close(); + } + } + + /// See Web.asciiLower; HTTP/2 names are lower case on the wire already, and + /// this keeps a JDK that hands them back otherwise from changing that. + private static String asciiLower(String value) { + StringBuilder out = new StringBuilder(value.length()); + for (int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char) (c + 32) : c); + } + return out.toString(); + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Web.java b/vm/backend/impl/javase/com/codename1/backend/Web.java index 7c012a725f9..b6c8665f196 100644 --- a/vm/backend/impl/javase/com/codename1/backend/Web.java +++ b/vm/backend/impl/javase/com/codename1/backend/Web.java @@ -245,6 +245,8 @@ interface TransportFactory { /// JDK's spelling. private static final class Transport { private final TrackingSockets sockets; + /// The java.net.http clients a WebClient that requires HTTP/2 uses. + private final JdkHttp2.Clients http2 = new JdkHttp2.Clients(); Transport(SSLSocketFactory sockets) { this.sockets = new TrackingSockets(sockets); @@ -387,20 +389,11 @@ static void closeTransport(Object transport) { // http:// connection has no factory to track it by and expires there // on its own; it carries no credential. ((Transport) transport).sockets.closeAll(); + ((Transport) transport).http2.close(); } static Result request(WebClient client, String method, String url, List headers, byte[] body) throws IOException { - if (client != null && client.isRequireHttp2()) { - // Said out loud, the way PATCH is: the request would otherwise go - // out over HTTP/1.1, the protocol the caller ruled out. - throw new IOException("The local Java SE runtime cannot make an HTTP/2 " - + "request -- its client is HttpURLConnection, which speaks HTTP/1.1 " - + "only. The packaged backend negotiates HTTP/2, so this is a " - + "limitation of cn1:backend rather than of your code. Exercise " - + "this path against the packaged binary, or leave HTTP/2 " - + "preferred rather than required while developing."); - } Urls.requireHttp(url); // The request line's other field; see HeaderLines.requireMethod. HeaderLines.requireMethod(method); @@ -453,6 +446,12 @@ private static String protocolOf(String statusLine) { /// the translated twin's rule, for the translated twin's reason. private static Result perform(WebClient client, String method, String url, List headers, byte[] body, List trace) throws IOException { + if (client != null && client.isRequireHttp2()) { + // HttpURLConnection speaks HTTP/1.1 only; the JDK's other client + // negotiates HTTP/2 and is refused if it does not get it. + return JdkHttp2.perform(client, ((Transport) client.transport(TRANSPORTS)).http2, + method, url, headers, body, trace, maxResponseBytes()); + } int connectTimeout = 30000; int readTimeout = 30000; int requestTimeout = 0; @@ -719,8 +718,8 @@ public static Exchange open(String method, String url, List headers, long reques static Exchange open(WebClient client, String method, String url, List headers, long requestLength) throws IOException { if (client != null && client.isRequireHttp2()) { - throw new IOException("The local Java SE runtime cannot make an HTTP/2 " - + "request; see Web.request"); + throw new IOException("The local Java SE runtime cannot stream an HTTP/2 " + + "exchange; Web.request makes the whole request over HTTP/2 instead"); } Urls.requireHttp(url); HeaderLines.requireMethod(method); diff --git a/vm/backend/src/com/codename1/backend/WebClient.java b/vm/backend/src/com/codename1/backend/WebClient.java index 0f45e63c4ba..0560c999095 100644 --- a/vm/backend/src/com/codename1/backend/WebClient.java +++ b/vm/backend/src/com/codename1/backend/WebClient.java @@ -61,10 +61,11 @@ /// IOException saying so, and the request is not sent to it as HTTP/1.1. Without /// it the packaged binary offers HTTP/2 over TLS and falls back to HTTP/1.1 when /// the server declines; [Web.Result#getProtocol()] says which was used. The local -/// Java SE runtime speaks HTTP/1.1 only (its client is HttpURLConnection, the -/// one every JDK this runs on has), so there a client that REQUIRES HTTP/2 fails -/// every request with an IOException that says so, rather than quietly -/// succeeding over a protocol the caller ruled out. +/// Java SE runtime makes an ordinary request with HttpURLConnection, which speaks +/// HTTP/1.1 only, and a client that REQUIRES HTTP/2 with the JDK's +/// `java.net.http` client (Java 11 or newer), so APNs and other HTTP/2-only +/// services work in the development loop and in tests. There, [#open(String, String, List, long)] -- a +/// streamed exchange -- is still HTTP/1.1 only and refuses such a client. public final class WebClient { private boolean requireHttp2; private ClientCertificate clientCertificate; From 32e908e435ecfef705056a3ce72647849a0c21dd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:12:23 +0200 Subject: [PATCH 08/25] Backend security: legacy recovery-code hashes, remember-me devices, configurable table names Recovery codes are verified through a DelegatingPasswordEncoder: new codes are {pbkdf2-sha256}, imported {sha256} hex hashes verify in constant time, and unprefixed hashes stay PBKDF2. Remember-me tokens record user agent and client address (schema version 12), and repositories list a user's tokens and revoke one by owner. Every cn1_* table name can be configured through cn1.security.table. or cn1.security.table-prefix, validated as a safe SQL identifier. Refs #6153 (item 9) Co-Authored-By: Claude Opus 5.5 --- .../developer-guide/Backend-Security.asciidoc | 31 +++ .../codename1/backend/security/MfaTest.java | 2 +- .../security/SecuritySchemaHistoryTest.java | 129 +++++++++- .../backend/security/SecurityStoresTest.java | 6 +- .../codename1/backend/security/Access.java | 16 ++ .../backend/security/SecuritySchema.java | 226 +++++++++++++++--- .../security/apikey/JdbcApiKeyRepository.java | 11 +- .../userdetails/JdbcUserDetailsManager.java | 46 ++-- .../crypto/Sha256HexPasswordEncoder.java | 84 +++++++ .../mfa/JdbcRecoveryCodeRepository.java | 10 +- .../security/mfa/JdbcTotpRepository.java | 16 +- .../security/mfa/RecoveryCodeService.java | 42 +++- .../JdbcFederatedIdentityRepository.java | 8 +- .../JdbcOAuth2AuthorizationService.java | 49 ++-- .../JdbcRegisteredClientRepository.java | 9 +- .../security/ratelimit/JdbcRateLimiter.java | 15 +- .../InMemoryTokenRepositoryImpl.java | 23 +- .../rememberme/JdbcTokenRepository.java | 79 ++++-- .../rememberme/PersistentRememberMeToken.java | 37 +++ ...ersistentTokenBasedRememberMeServices.java | 11 +- .../rememberme/PersistentTokenRepository.java | 32 +++ ...blicKeyCredentialUserEntityRepository.java | 8 +- .../JdbcUserCredentialRepository.java | 11 +- 23 files changed, 758 insertions(+), 143 deletions(-) create mode 100644 vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java diff --git a/docs/developer-guide/Backend-Security.asciidoc b/docs/developer-guide/Backend-Security.asciidoc index 5963788ec96..786c2c09975 100644 --- a/docs/developer-guide/Backend-Security.asciidoc +++ b/docs/developer-guide/Backend-Security.asciidoc @@ -1347,6 +1347,13 @@ Show them to the user once. The server keeps a hash of each and can't show them again. Each code works once, in place of a one-time code at sign-in, and generating again replaces whatever was left. +New codes are hashed with PBKDF2 and stored as `{pbkdf2-sha256}...`. A stored +hash says which scheme made it, the way a password does, so codes another +system issued can be imported without asking every user for new ones: store +the hexadecimal SHA-256 of each code as `{sha256}`, hashed from the code in +lower case without its dashes or spaces. To verify another scheme, pass your +own `PasswordEncoder` to `new RecoveryCodeService(repository, encoder)`. + ==== The encryption key `JdbcTotpRepository` seals each authenticator secret with AES-GCM before it @@ -1506,6 +1513,11 @@ On a chain with `mfa`, the cookie of a user who has a second factor works only if the sign-in that issued it passed that factor. The server records which kind each cookie is. +Each token records the `User-Agent` and client address of the sign-in that +issued it. `PersistentTokenRepository.getTokensForUser` lists a user's tokens +for a "signed-in devices" page, and `removeUserToken(username, series)` signs +one of them out. It refuses a series that belongs to someone else. + Without a `tokenRepository` or a `PersistentTokenRepository` bean, the tokens are kept in memory. They're then forgotten when the server restarts, and they're no use to a deployment of several instances. @@ -1606,8 +1618,27 @@ are the same on SQLite, PostgreSQL, and MySQL. | 11 | `cn1_persistent_logins.previous_token_hash` | A brief grace period for parallel uses of a remembered sign-in after token rotation. + +| 12 +| `cn1_persistent_logins.user_agent`, `cn1_persistent_logins.ip_address` +| The device each remembered sign-in came from. |=== +To use tables you already have, give their names in `application.properties`. +A per-table setting names one table, and a prefix replaces the `cn1_` in front +of all of them. The per-table setting wins: + +[source,properties] +---- +cn1.security.table-prefix=auth_ +cn1.security.table.cn1_users=app_users +---- + +A name must be letters, digits and `_`, optionally after one schema name and a +dot (`auth.users`). The server refuses to start with anything else. An adopted +table must have the columns this chapter's stores use. Every store and every +schema version uses the configured names. + The key creates the tables. It doesn't choose the stores. Each store is still a bean you declare, or an object you hand to a chain, as the examples in this chapter do. A feature whose store you don't declare keeps its state in memory diff --git a/maven/backend/src/test/java/com/codename1/backend/security/MfaTest.java b/maven/backend/src/test/java/com/codename1/backend/security/MfaTest.java index eb4b7446eae..93ef02b3eac 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/MfaTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/MfaTest.java @@ -262,7 +262,7 @@ void recoveryCodesUseSaltedPasswordHashesAndStillConsumeAtomically() throws Exce assertEquals(10, hashes.size()); java.util.Set salts = new java.util.HashSet(); for (String hash : hashes) { - assertTrue(hash.startsWith("pbkdf2$"), hash); + assertTrue(hash.startsWith("{pbkdf2-sha256}pbkdf2$"), hash); salts.add(hash.split("\\$")[2]); } assertEquals(10, salts.size()); diff --git a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java index 60937ff64c6..73ffb4a563d 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.codename1.backend.DataSource; @@ -94,7 +95,7 @@ void upgradingRememberMeKeepsExistingTokens() throws Exception { Migrations.of(pool, SecuritySchema.migrations()).target("10").migrate(); pool.execute("INSERT INTO cn1_persistent_logins (series, username_key, username, token_hash, last_used) " + "VALUES ('browser', 'ada', 'Ada', 'old-hash', 1000)", null); - assertEquals(1, Migrations.of(pool, SecuritySchema.migrations()).migrate().getMigrationsExecuted()); + assertEquals(2, Migrations.of(pool, SecuritySchema.migrations()).migrate().getMigrationsExecuted()); com.codename1.backend.security.rememberme.JdbcTokenRepository repository = new com.codename1.backend.security.rememberme.JdbcTokenRepository(pool); assertEquals("old-hash", repository.getTokenForSeries("browser").getTokenHash()); @@ -112,9 +113,9 @@ void everyVersionHasItsOwnName() throws Exception { DataSource pool = DataSource.open(new File(dir, "named.db").getPath(), 2, 5000, 10000); try { MigrateResult result = Migrations.of(pool, SecuritySchema.migrations()).migrate(); - assertEquals(11, result.getMigrationsExecuted()); - assertEquals(11, new TreeSet(result.getApplied()).size(), - "eleven versions reported under fewer names: " + result.getApplied()); + assertEquals(12, result.getMigrationsExecuted()); + assertEquals(12, new TreeSet(result.getApplied()).size(), + "twelve versions reported under fewer names: " + result.getApplied()); assertEquals("com.codename1.backend.security.SecuritySchema.V1__users_and_authorities", result.getApplied().get(0)); assertEquals("com.codename1.backend.security.SecuritySchema.V9__passkeys", @@ -139,9 +140,9 @@ void aHistoryWrittenBeforeTheNamesStillValidates() throws Exception { Migrations.of(pool, SecuritySchema.migrations()).validate(); MigrateResult again = Migrations.of(pool, SecuritySchema.migrations()).migrate(); assertEquals(0, again.getMigrationsExecuted()); - assertEquals("11", again.getTargetVersion()); + assertEquals("12", again.getTargetVersion()); MigrationInfo[] info = Migrations.of(pool, SecuritySchema.migrations()).info(); - assertEquals(11, info.length); + assertEquals(12, info.length); for (MigrationInfo row : info) { assertEquals(MigrationState.SUCCESS, row.getState(), "version " + row.getVersion()); } @@ -151,4 +152,120 @@ void aHistoryWrittenBeforeTheNamesStillValidates() throws Exception { pool.close(); } } + + @Test + void rememberMeRecordsAndListsTheDeviceOnAnUpgradedTable() throws Exception { + DataSource pool = DataSource.open(new File(dir, "item9-devices.db").getPath(), 2, 5000, 10000); + try { + // A table as version 11 left it: no device columns. + Migrations.of(pool, SecuritySchema.migrations()).target("11").migrate(); + pool.execute("INSERT INTO cn1_persistent_logins (series, username_key, username, " + + "token_hash, last_used) VALUES ('old', 'ada', 'Ada', 'h0', 1000)", null); + assertEquals(1, Migrations.of(pool, SecuritySchema.migrations()).migrate() + .getMigrationsExecuted()); + com.codename1.backend.security.rememberme.JdbcTokenRepository repository = + new com.codename1.backend.security.rememberme.JdbcTokenRepository(pool); + repository.createNewToken(new com.codename1.backend.security.rememberme + .PersistentRememberMeToken("Ada", "phone", "h1", 3000L, null, "Phone/1.0", + "203.0.113.7")); + repository.createNewToken(new com.codename1.backend.security.rememberme + .PersistentRememberMeToken("Bob", "bobs", "h2", 2000L)); + java.util.List + devices = repository.getTokensForUser("ADA"); + assertEquals(2, devices.size()); + assertEquals("phone", devices.get(0).getSeries()); + assertEquals("Phone/1.0", devices.get(0).getUserAgent()); + assertEquals("203.0.113.7", devices.get(0).getIpAddress()); + assertEquals(null, devices.get(1).getUserAgent()); + assertEquals("Phone/1.0", repository.getTokenForSeries("phone").getUserAgent()); + // Revoking one series names its owner. + assertFalse(repository.removeUserToken("Bob", "phone")); + assertTrue(repository.removeUserToken("ada", "phone")); + assertEquals(1, repository.getTokensForUser("Ada").size()); + assertEquals(1, repository.getTokensForUser("Bob").size()); + } finally { + pool.close(); + } + } + + @Test + void configuredTableNamesAreUsedEverywhere() throws Exception { + DataSource pool = DataSource.open(new File(dir, "item9-names.db").getPath(), 2, 5000, 10000); + java.util.Properties props = new java.util.Properties(); + props.setProperty(SecuritySchema.TABLE_NAME_PREFIX, "app_"); + props.setProperty(SecuritySchema.TABLE_PREFIX + "cn1_persistent_logins", "remembered"); + try { + SecuritySchema.configure(com.codename1.backend.Config.of(props, "default")); + assertEquals("app_users", SecuritySchema.table("cn1_users")); + assertEquals("remembered", SecuritySchema.table("cn1_persistent_logins")); + Migrations.of(pool, SecuritySchema.migrations()).migrate(); + assertEquals(0L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM app_mfa_recovery_code", + null).get("n")).longValue()); + com.codename1.backend.security.rememberme.JdbcTokenRepository tokens = + new com.codename1.backend.security.rememberme.JdbcTokenRepository(pool); + tokens.createNewToken(new com.codename1.backend.security.rememberme + .PersistentRememberMeToken("Ada", "s", "h", 1L)); + assertEquals(1L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM remembered", + null).get("n")).longValue()); + com.codename1.backend.security.mfa.RecoveryCodeService codes = + new com.codename1.backend.security.mfa.RecoveryCodeService( + new com.codename1.backend.security.mfa.JdbcRecoveryCodeRepository(pool)); + codes.generate("Ada"); + assertEquals(10L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM app_mfa_recovery_code", + null).get("n")).longValue()); + assertThrows(IllegalArgumentException.class, + () -> SecuritySchema.setTableName("cn1_users", "users; DROP TABLE x")); + assertThrows(IllegalArgumentException.class, + () -> SecuritySchema.setTableName("cn1_users", "1users")); + assertThrows(IllegalArgumentException.class, + () -> SecuritySchema.setTableName("cn1_users", "a.b.c")); + SecuritySchema.setTableName("cn1_users", "auth.users"); + assertEquals("auth.users", SecuritySchema.table("cn1_users")); + java.util.Properties bad = new java.util.Properties(); + bad.setProperty(SecuritySchema.TABLE_PREFIX + "cn1_users", "x\"y"); + assertThrows(IllegalArgumentException.class, + () -> SecuritySchema.configure(com.codename1.backend.Config.of(bad, "default"))); + } finally { + SecuritySchema.resetTableNames(); + pool.close(); + } + } + + @Test + void legacySha256RecoveryCodesVerifyOnce() throws Exception { + DataSource pool = DataSource.open(new File(dir, "item9-codes.db").getPath(), 2, 5000, 10000); + try { + Migrations.of(pool, SecuritySchema.migrations()).migrate(); + com.codename1.backend.security.mfa.JdbcRecoveryCodeRepository repository = + new com.codename1.backend.security.mfa.JdbcRecoveryCodeRepository(pool); + com.codename1.backend.security.crypto.Sha256HexPasswordEncoder sha = + new com.codename1.backend.security.crypto.Sha256HexPasswordEncoder(); + // Imported from another system: upper case hex of the normalized code. + java.util.List imported = new java.util.ArrayList(); + imported.add("{sha256}" + sha.encode("abcde23456").toUpperCase(java.util.Locale.ROOT)); + imported.add("{sha256}" + sha.encode("zzzzz99999")); + repository.replace("Ada", imported); + com.codename1.backend.security.mfa.RecoveryCodeService codes = + new com.codename1.backend.security.mfa.RecoveryCodeService(repository); + assertFalse(codes.consume("Ada", "abcde-23457")); + assertTrue(codes.consume("Ada", "ABCDE-23456")); + assertFalse(codes.consume("Ada", "abcde-23456")); + assertEquals(1, codes.remaining("Ada")); + // Codes made here are PBKDF2, and say so. + java.util.List fresh = codes.generate("Ada"); + for (String hash : repository.findHashes("Ada")) { + assertTrue(hash.startsWith("{pbkdf2-sha256}"), hash); + } + assertTrue(codes.consume("Ada", fresh.get(3))); + assertFalse(codes.consume("Ada", fresh.get(3))); + // A hash stored before ids were written is still PBKDF2. + java.util.List bare = new java.util.ArrayList(); + bare.add(new com.codename1.backend.security.crypto.Pbkdf2Sha256PasswordEncoder() + .encode("mnpqr45678")); + repository.replace("Bob", bare); + assertTrue(codes.consume("Bob", "mnpqr-45678")); + } finally { + pool.close(); + } + } } diff --git a/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java b/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java index b12e5576f4e..7baadb47f7f 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java @@ -148,7 +148,7 @@ void theSchemaIsAppliedOnceUnderItsOwnHistory(String engine) throws Exception { } } MigrationInfo[] info = Migrations.of(pool, SecuritySchema.migrations()).info(); - assertEquals(11, info.length); + assertEquals(12, info.length); for (int i = 0; i < info.length; i++) { assertEquals(String.valueOf(i + 1), info[i].getVersion()); assertEquals(MigrationState.SUCCESS, info[i].getState()); @@ -161,12 +161,12 @@ void theSchemaIsAppliedOnceUnderItsOwnHistory(String engine) throws Exception { assertEquals("passkeys", info[8].getDescription()); assertEquals("rate limit expiry", info[9].getDescription()); // Its own history, not the application's. - assertEquals(11L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM " + assertEquals(12L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM " + "cn1_security_schema_history", null).get("n")).longValue()); // And a second run finds nothing to do. MigrateResult again = Migrations.of(pool, SecuritySchema.migrations()).migrate(); assertEquals(0, again.getMigrationsExecuted()); - assertEquals("11", again.getTargetVersion()); + assertEquals("12", again.getTargetVersion()); assertEquals("security", SecuritySchema.migrations().getName()); assertEquals("cn1_security_schema_history", SecuritySchema.migrations().getTable()); } diff --git a/vm/backend/src/com/codename1/backend/security/Access.java b/vm/backend/src/com/codename1/backend/security/Access.java index f22dbfe5762..d6cbb25d541 100644 --- a/vm/backend/src/com/codename1/backend/security/Access.java +++ b/vm/backend/src/com/codename1/backend/security/Access.java @@ -35,6 +35,7 @@ final class Access extends SecurityAccess { public HttpSecurity httpSecurity(Config config, Object[] beans) { com.codename1.impl.backend.security.SecuritySupport.developmentProfile( config != null && config.isDevelopmentProfile()); + tableNames(config); return new HttpSecurity(config, beans); } @@ -43,6 +44,7 @@ public HttpSecurity httpSecurity(Config config, Object[] beans, String[] names, boolean[] primary) { com.codename1.impl.backend.security.SecuritySupport.developmentProfile( config != null && config.isDevelopmentProfile()); + tableNames(config); return new HttpSecurity(config, beans, names, primary); } @@ -97,4 +99,18 @@ public String testCsrf(boolean valid) { public void clearTestCsrf() { CsrfFilter.testToken(null); } + + // The server's configuration names the security tables; read here so every + // repository uses them without a configuration of its own. + private static void tableNames(Config config) { + if (config == null) { + return; + } + try { + SecuritySchema.configure(config); + } catch (java.io.IOException err) { + throw new IllegalStateException("The security table names could not be read: " + + err.getMessage(), err); + } + } } diff --git a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java index 2be77408100..ca824e2b0b3 100644 --- a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java +++ b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java @@ -22,12 +22,17 @@ */ package com.codename1.backend.security; +import com.codename1.backend.Config; import com.codename1.backend.Database; import com.codename1.backend.sql.Dialect; import com.codename1.migration.JavaMigration; import com.codename1.migration.MigrationContext; import com.codename1.migration.MigrationSet; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; /// The tables the security layer's database-backed stores keep, as a migration /// set of the layer's own. @@ -75,6 +80,30 @@ /// | 9 | `cn1_webauthn_user`, `cn1_webauthn_credential` | [com.codename1.backend.security.webauthn.JdbcPublicKeyCredentialUserEntityRepository], [com.codename1.backend.security.webauthn.JdbcUserCredentialRepository] | /// | 10 | `cn1_rate_limit.window_end` | Expiry-aware cleanup for limiters with different periods | /// | 11 | `cn1_persistent_logins.previous_token_hash` | Concurrent remember-me rotation grace | +/// | 12 | `cn1_persistent_logins.user_agent`, `cn1_persistent_logins.ip_address` | The device a remember-me sign-in came from | +/// +/// #### Adopting an existing schema +/// +/// The names above are the defaults. Each one can be pointed at a table of +/// another name, so that tables an application already has are used in place +/// rather than copied: +/// +/// ```properties +/// cn1.security.table.cn1_users=app_users +/// cn1.security.table.cn1_persistent_logins=persistent_logins +/// ``` +/// +/// or every one renamed at once by a prefix that replaces `cn1_`: +/// +/// ```properties +/// cn1.security.table-prefix=auth_ +/// ``` +/// +/// A per-table setting wins over the prefix. A name must be a plain SQL +/// identifier (letters, digits and `_`, optionally qualified by one schema +/// name); anything else is refused when the settings are read. The columns of +/// an adopted table must be the ones described here. Every repository in this +/// package, and the migrations of this set, use the configured names. /// /// A user name is kept twice: as it was given, and folded to lower case in the /// `username_key` column every table is keyed by, which is what makes a lookup @@ -91,18 +120,152 @@ public final class SecuritySchema { // The versions of this set. A version is never edited once released: a new // table, or a change to one, is a new version appended in migrations(). - // The next takes 12. Each version is one method below that returns the + // The next takes 13. Each version is one method below that returns the // statements for an engine, a case in Tables.migrate, and a line in // migrations(). + /// The prefix of a per-table name setting: `cn1.security.table.`. + public static final String TABLE_PREFIX = "cn1.security.table."; + /// The setting that replaces the `cn1_` in front of every table name. + public static final String TABLE_NAME_PREFIX = "cn1.security.table-prefix"; + + private static final String[] TABLES = { + "cn1_users", "cn1_authorities", "cn1_api_key", "cn1_persistent_logins", "cn1_mfa_totp", + "cn1_mfa_recovery_code", "cn1_rate_limit", "cn1_federated_identity", + "cn1_oauth2_registered_client", "cn1_oauth2_authorization", "cn1_oauth2_token", + "cn1_webauthn_user", "cn1_webauthn_credential" + }; + + // Null until configured; then the default name -> configured name map. + private static final AtomicReference> NAMES = + new AtomicReference>(); + private SecuritySchema() { } + /// Reads the table name settings from `config`; see the class description. + /// The server calls this with its own configuration. Without it, the + /// settings are read from [Config#load()] the first time a name is needed. + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: a configured name is not a safe SQL identifier + public static void configure(Config config) throws IOException { + Map names = new HashMap(); + String prefix = config == null ? null : config.get(TABLE_NAME_PREFIX, null); + if (prefix != null && prefix.trim().length() > 0) { + prefix = prefix.trim(); + } else { + prefix = null; + } + for (String table : TABLES) { + String name = config == null ? null : config.get(TABLE_PREFIX + table, null); + if (name == null || name.trim().length() == 0) { + name = prefix == null ? table : prefix + table.substring(4); + } + names.put(table, checkIdentifier(name.trim())); + } + NAMES.set(Collections.unmodifiableMap(names)); + } + + /// Points one table at another name, as the `cn1.security.table.` + /// setting does. + /// + /// #### Parameters + /// + /// - `defaultName`: the table's name in this class, such as `cn1_users` + /// - `name`: the table to use instead, or null for the default + public static void setTableName(String defaultName, String name) { + if (indexOf(defaultName) < 0) { + throw new IllegalArgumentException("The security schema has no table " + defaultName); + } + String checked = name == null ? defaultName : checkIdentifier(name); + while (true) { + Map current = names(); + Map next = new HashMap(current); + next.put(defaultName, checked); + if (NAMES.compareAndSet(current, Collections.unmodifiableMap(next))) { + return; + } + } + } + + /// Restores every default name. + public static void resetTableNames() { + Map names = new HashMap(); + for (String table : TABLES) { + names.put(table, table); + } + NAMES.set(Collections.unmodifiableMap(names)); + } + + /// The name a table is used under, after the settings. + /// + /// #### Parameters + /// + /// - `defaultName`: the table's name in this class, such as `cn1_users` + public static String table(String defaultName) { + String name = names().get(defaultName); + if (name == null) { + throw new IllegalArgumentException("The security schema has no table " + defaultName); + } + return name; + } + + private static Map names() { + Map names = NAMES.get(); + if (names == null) { + try { + configure(Config.load()); + } catch (IOException err) { + throw new IllegalStateException("The security table names could not be read: " + + err.getMessage(), err); + } + names = NAMES.get(); + } + return names; + } + + private static int indexOf(String table) { + for (int iter = 0 ; iter < TABLES.length ; iter++) { + if (TABLES[iter].equals(table)) { + return iter; + } + } + return -1; + } + + /// A name that can be written into SQL as it is: one or two dot-separated + /// parts, each a letter or `_` followed by letters, digits and `_`. + static String checkIdentifier(String name) { + boolean ok = name != null && name.length() > 0 && name.length() <= 128; + boolean start = true; + int dots = 0; + for (int iter = 0 ; ok && iter < name.length() ; iter++) { + char c = name.charAt(iter); + boolean letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + if (c == '.') { + ok = !start && ++dots == 1; + start = true; + } else if (start) { + ok = letter; + start = false; + } else { + ok = letter || (c >= '0' && c <= '9'); + } + } + if (!ok || start) { + throw new IllegalArgumentException("Not a safe SQL table name: " + name); + } + return name; + } + /// What each version adds, in version order: the description its history row carries. private static final String[] DESCRIPTIONS = { "users and authorities", "api keys", "persistent logins", "second factors", "rate limits", "federated identities", "oauth2 registered clients", - "oauth2 authorizations", "passkeys", "rate limit expiry", "remember me rotation grace" + "oauth2 authorizations", "passkeys", "rate limit expiry", "remember me rotation grace", + "remember me devices" }; /// The set, for `Migrations.register`. @@ -159,6 +322,7 @@ public void migrate(MigrationContext context) throws IOException { case 9: statements = passkeys(d); break; case 10: statements = rateLimitExpiry(d); break; case 11: statements = rememberMeRotationGrace(); break; + case 12: statements = rememberMeDevices(d); break; default: throw new IOException("The security schema has no version " + version); } for (String statement : statements) { @@ -212,44 +376,44 @@ private static String moment(Dialect d) { private static String[] users(Dialect d) { return new String[] { - "CREATE TABLE cn1_users (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_users") + " (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "username " + text(d) + ", password " + text(d) + ", enabled " + flag(d) + ", account_non_expired " + flag(d) + ", account_non_locked " + flag(d) + ", credentials_non_expired " + flag(d) + ")", - "CREATE TABLE cn1_authorities (username_key " + key(d) + " NOT NULL, " + "CREATE TABLE " + table("cn1_authorities") + " (username_key " + key(d) + " NOT NULL, " + "authority " + key(d) + " NOT NULL, PRIMARY KEY (username_key, authority))"}; } private static String[] apiKeys(Dialect d) { return new String[] { - "CREATE TABLE cn1_api_key (key_hash " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_api_key") + " (key_hash " + key(d) + " NOT NULL PRIMARY KEY, " + "id " + key(d) + " NOT NULL, owner " + key(d) + " NOT NULL, scopes " + text(d) + ", prefix " + text(d) + ", last_four " + text(d) + ", revoked " + flag(d) + ", created_at " + moment(d) + ")", - "CREATE UNIQUE INDEX cn1_api_key_id ON cn1_api_key (id)", - "CREATE INDEX cn1_api_key_owner ON cn1_api_key (owner)"}; + "CREATE UNIQUE INDEX cn1_api_key_id ON " + table("cn1_api_key") + " (id)", + "CREATE INDEX cn1_api_key_owner ON " + table("cn1_api_key") + " (owner)"}; } private static String[] persistentLogins(Dialect d) { return new String[] { - "CREATE TABLE cn1_persistent_logins (series " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_persistent_logins") + " (series " + key(d) + " NOT NULL PRIMARY KEY, " + "username_key " + key(d) + " NOT NULL, username " + text(d) + ", token_hash " + text(d) + ", last_used " + moment(d) + ")", - "CREATE INDEX cn1_persistent_logins_user ON cn1_persistent_logins (username_key)"}; + "CREATE INDEX cn1_persistent_logins_user ON " + table("cn1_persistent_logins") + " (username_key)"}; } private static String[] secondFactors(Dialect d) { return new String[] { - "CREATE TABLE cn1_mfa_totp (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_mfa_totp") + " (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "secret " + text(d) + ", nonce " + text(d) + ", confirmed " + flag(d) + ", last_used_step " + moment(d) + ", created_at " + moment(d) + ")", - "CREATE TABLE cn1_mfa_recovery_code (username_key " + key(d) + " NOT NULL, " + "CREATE TABLE " + table("cn1_mfa_recovery_code") + " (username_key " + key(d) + " NOT NULL, " + "code_hash " + key(d) + " NOT NULL, PRIMARY KEY (username_key, code_hash))"}; } private static String[] rateLimits(Dialect d) { return new String[] { - "CREATE TABLE cn1_rate_limit (limit_key " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_rate_limit") + " (limit_key " + key(d) + " NOT NULL PRIMARY KEY, " + "window_start " + moment(d) + ", hits " + d.columnType(Dialect.INTEGER) + " NOT NULL)"}; } @@ -257,30 +421,38 @@ private static String[] rateLimits(Dialect d) { private static String[] rateLimitExpiry(Dialect d) { // Old rows have no known period. Keep them until their next window starts; // guessing their expiry could reset an active limiter during an upgrade. - return new String[] {"ALTER TABLE cn1_rate_limit ADD COLUMN window_end " + d.columnType(Dialect.BIGINT)}; + return new String[] {"ALTER TABLE " + table("cn1_rate_limit") + " ADD COLUMN window_end " + d.columnType(Dialect.BIGINT)}; } private static String[] rememberMeRotationGrace() { - return new String[] {"ALTER TABLE cn1_persistent_logins ADD COLUMN previous_token_hash VARCHAR(64)"}; + return new String[] {"ALTER TABLE " + table("cn1_persistent_logins") + " ADD COLUMN previous_token_hash VARCHAR(64)"}; + } + + private static String[] rememberMeDevices(Dialect d) { + // Nullable: tokens issued before this version have no device recorded. + String type = d.columnType(Dialect.TEXT); + return new String[] { + "ALTER TABLE " + table("cn1_persistent_logins") + " ADD COLUMN user_agent " + type, + "ALTER TABLE " + table("cn1_persistent_logins") + " ADD COLUMN ip_address " + type}; } private static String[] federatedIdentities(Dialect d) { return new String[] { - "CREATE TABLE cn1_federated_identity (provider " + key(d) + " NOT NULL, subject " + "CREATE TABLE " + table("cn1_federated_identity") + " (provider " + key(d) + " NOT NULL, subject " + key(d) + " NOT NULL, username_key " + key(d) + " NOT NULL, username " + text(d) + ", created_at " + moment(d) + ", PRIMARY KEY (provider, subject))", - "CREATE INDEX cn1_federated_identity_user ON cn1_federated_identity (username_key)"}; + "CREATE INDEX cn1_federated_identity_user ON " + table("cn1_federated_identity") + " (username_key)"}; } private static String[] registeredClients(Dialect d) { return new String[] { - "CREATE TABLE cn1_oauth2_registered_client (id " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_oauth2_registered_client") + " (id " + key(d) + " NOT NULL PRIMARY KEY, " + "client_id " + key(d) + " NOT NULL, client_secret " + text(d) + ", client_name " + text(d) + ", authentication_methods " + text(d) + ", grant_types " + text(d) + ", redirect_uris " + text(d) + ", scopes " + text(d) + ", settings " + text(d) + ", created_at " + moment(d) + ")", "CREATE UNIQUE INDEX cn1_oauth2_registered_client_cid ON " - + "cn1_oauth2_registered_client (client_id)"}; + + table("cn1_oauth2_registered_client") + " (client_id)"}; } /// A grant, and beside it the secrets issued under it -- the authorization @@ -288,17 +460,17 @@ private static String[] registeredClients(Dialect d) { /// keyed by its SHA-256, so that using one up is one statement on one row. private static String[] authorizations(Dialect d) { return new String[] { - "CREATE TABLE cn1_oauth2_authorization (id " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_oauth2_authorization") + " (id " + key(d) + " NOT NULL PRIMARY KEY, " + "registered_client_id " + key(d) + " NOT NULL, principal_name " + text(d) + ", grant_type " + text(d) + ", scopes " + text(d) + ", status " + key(d) + " NOT NULL, attributes " + text(d) + ", created_at " + moment(d) + ", expires_at " + moment(d) + ")", - "CREATE INDEX cn1_oauth2_authorization_exp ON cn1_oauth2_authorization (expires_at)", - "CREATE TABLE cn1_oauth2_token (token_hash " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE INDEX cn1_oauth2_authorization_exp ON " + table("cn1_oauth2_authorization") + " (expires_at)", + "CREATE TABLE " + table("cn1_oauth2_token") + " (token_hash " + key(d) + " NOT NULL PRIMARY KEY, " + "authorization_id " + key(d) + " NOT NULL, kind " + key(d) + " NOT NULL, used " + flag(d) + ", expires_at " + moment(d) + ", polled_at " + moment(d) + ")", - "CREATE INDEX cn1_oauth2_token_auth ON cn1_oauth2_token (authorization_id)", - "CREATE INDEX cn1_oauth2_token_exp ON cn1_oauth2_token (expires_at)"}; + "CREATE INDEX cn1_oauth2_token_auth ON " + table("cn1_oauth2_token") + " (authorization_id)", + "CREATE INDEX cn1_oauth2_token_exp ON " + table("cn1_oauth2_token") + " (expires_at)"}; } /// A user's passkey handle, and their credentials. Byte strings -- the @@ -308,17 +480,17 @@ private static String[] authorizations(Dialect d) { /// the table is keyed by its SHA-256 instead and the id is a column. private static String[] passkeys(Dialect d) { return new String[] { - "CREATE TABLE cn1_webauthn_user (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "CREATE TABLE " + table("cn1_webauthn_user") + " (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "username " + text(d) + ", user_id " + key(d) + " NOT NULL, display_name " + text(d) + ", created_at " + moment(d) + ")", - "CREATE UNIQUE INDEX cn1_webauthn_user_id ON cn1_webauthn_user (user_id)", - "CREATE TABLE cn1_webauthn_credential (credential_key " + key(d) + "CREATE UNIQUE INDEX cn1_webauthn_user_id ON " + table("cn1_webauthn_user") + " (user_id)", + "CREATE TABLE " + table("cn1_webauthn_credential") + " (credential_key " + key(d) + " NOT NULL PRIMARY KEY, credential_id " + text(d) + ", user_id " + key(d) + " NOT NULL, algorithm " + moment(d) + ", public_key " + text(d) + ", sign_count " + moment(d) + ", uv_initialized " + flag(d) + ", backup_eligible " + flag(d) + ", backup_state " + flag(d) + ", transports " + text(d) + ", label " + text(d) + ", created_at " + moment(d) + ", last_used " + moment(d) + ")", - "CREATE INDEX cn1_webauthn_credential_user ON cn1_webauthn_credential (user_id)"}; + "CREATE INDEX cn1_webauthn_credential_user ON " + table("cn1_webauthn_credential") + " (user_id)"}; } } diff --git a/vm/backend/src/com/codename1/backend/security/apikey/JdbcApiKeyRepository.java b/vm/backend/src/com/codename1/backend/security/apikey/JdbcApiKeyRepository.java index ee4566a3d43..b441598e37d 100644 --- a/vm/backend/src/com/codename1/backend/security/apikey/JdbcApiKeyRepository.java +++ b/vm/backend/src/com/codename1/backend/security/apikey/JdbcApiKeyRepository.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security.apikey; +import com.codename1.backend.security.SecuritySchema; import com.codename1.backend.DataSource; import com.codename1.backend.security.Clock; import java.io.IOException; @@ -81,7 +82,7 @@ public ApiKey findByHash(String hash) { return null; } try { - Map row = dataSource.queryOne("SELECT " + COLUMNS + " FROM cn1_api_key WHERE " + Map row = dataSource.queryOne("SELECT " + COLUMNS + " FROM " + SecuritySchema.table("cn1_api_key") + " WHERE " + "key_hash = ?", new Object[] {hash}); return row == null ? null : read(row); } catch (IOException err) { @@ -104,7 +105,7 @@ public void save(ApiKey key) throws IOException { } scopes.append(scopes.length() == 0 ? "" : " ").append(scope); } - dataSource.execute("INSERT INTO cn1_api_key (key_hash, id, owner, scopes, prefix, " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_api_key") + " (key_hash, id, owner, scopes, prefix, " + "last_four, revoked, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", new Object[] { key.getHash(), key.getId(), key.getOwner(), scopes.toString(), key.getPrefix(), key.getLastFour(), Integer.valueOf(key.isRevoked() ? 1 : 0), @@ -116,19 +117,19 @@ public void save(ApiKey key) throws IOException { /// @return whether this call revoked it: false when there is no such key, or /// it was revoked already public boolean revoke(String id) throws IOException { - return dataSource.execute("UPDATE cn1_api_key SET revoked = 1 WHERE id = ? AND " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_api_key") + " SET revoked = 1 WHERE id = ? AND " + "revoked = 0", new Object[] {id}) == 1; } /// Forgets the key with this id altogether. public boolean delete(String id) throws IOException { - return dataSource.execute("DELETE FROM cn1_api_key WHERE id = ?", new Object[] {id}) == 1; + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_api_key") + " WHERE id = ?", new Object[] {id}) == 1; } /// The keys of one owner, oldest first. public List findByOwner(String owner) throws IOException { List out = new ArrayList(); - for (Object row : dataSource.query("SELECT " + COLUMNS + " FROM cn1_api_key WHERE " + for (Object row : dataSource.query("SELECT " + COLUMNS + " FROM " + SecuritySchema.table("cn1_api_key") + " WHERE " + "owner = ? ORDER BY created_at, id", new Object[] {owner})) { out.add(read((Map) row)); } diff --git a/vm/backend/src/com/codename1/backend/security/core/userdetails/JdbcUserDetailsManager.java b/vm/backend/src/com/codename1/backend/security/core/userdetails/JdbcUserDetailsManager.java index ae827345033..c0d224b0844 100644 --- a/vm/backend/src/com/codename1/backend/security/core/userdetails/JdbcUserDetailsManager.java +++ b/vm/backend/src/com/codename1/backend/security/core/userdetails/JdbcUserDetailsManager.java @@ -87,12 +87,12 @@ public UserDetails loadUserByUsername(String username) { List authorities; try { row = dataSource.queryOne("SELECT username, password, enabled, account_non_expired, " - + "account_non_locked, credentials_non_expired FROM cn1_users WHERE " + + "account_non_locked, credentials_non_expired FROM " + SecuritySchema.table("cn1_users") + " WHERE " + "username_key = ?", new Object[] {key}); if (row == null) { throw new UsernameNotFoundException(username); } - authorities = dataSource.query("SELECT authority FROM cn1_authorities WHERE " + authorities = dataSource.query("SELECT authority FROM " + SecuritySchema.table("cn1_authorities") + " WHERE " + "username_key = ?", new Object[] {key}); } catch (java.io.IOException err) { throw failed("load the user", err); @@ -120,7 +120,7 @@ private static void insertAuthorities(Database db, String key, UserDetails user) String name = authority.getAuthority(); if (name != null && !seen.contains(name)) { seen.add(name); - db.execute("INSERT INTO cn1_authorities (username_key, authority) VALUES (?, ?)", + db.execute("INSERT INTO " + SecuritySchema.table("cn1_authorities") + " (username_key, authority) VALUES (?, ?)", new Object[] {key, name}); } } @@ -174,11 +174,11 @@ private static final class Change implements DataSource.Work { @Override public Object run(Database db) throws Exception { if (kind == CREATE) { - if (db.queryOne("SELECT username_key FROM cn1_users WHERE username_key = ?", + if (db.queryOne("SELECT username_key FROM " + SecuritySchema.table("cn1_users") + " WHERE username_key = ?", new Object[] {key}) != null) { throw new IllegalArgumentException("user should not exist"); } - db.execute("INSERT INTO cn1_users (username_key, username, password, enabled, " + db.execute("INSERT INTO " + SecuritySchema.table("cn1_users") + " (username_key, username, password, enabled, " + "account_non_expired, account_non_locked, credentials_non_expired) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", new Object[] {key, user.getUsername(), user.getPassword(), flag(user.isEnabled()), @@ -188,7 +188,7 @@ public Object run(Database db) throws Exception { return null; } if (kind == UPDATE) { - int changed = db.execute("UPDATE cn1_users SET username = ?, password = ?, " + int changed = db.execute("UPDATE " + SecuritySchema.table("cn1_users") + " SET username = ?, password = ?, " + "enabled = ?, account_non_expired = ?, account_non_locked = ?, " + "credentials_non_expired = ? WHERE username_key = ?", new Object[] {user.getUsername(), user.getPassword(), @@ -198,45 +198,45 @@ public Object run(Database db) throws Exception { if (changed != 1) { throw new IllegalArgumentException("user should exist"); } - db.execute("DELETE FROM cn1_authorities WHERE username_key = ?", + db.execute("DELETE FROM " + SecuritySchema.table("cn1_authorities") + " WHERE username_key = ?", new Object[] {key}); insertAuthorities(db, key, user); return null; } deleteCredentials(db, key); - db.execute("DELETE FROM cn1_authorities WHERE username_key = ?", new Object[] {key}); - db.execute("DELETE FROM cn1_users WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_authorities") + " WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_users") + " WHERE username_key = ?", new Object[] {key}); return null; } /// These rows must disappear in the same transaction as the account. Otherwise /// reusing a username lets the old account's durable credentials sign in again. private static void deleteCredentials(Database db, String key) throws java.io.IOException { - db.execute("DELETE FROM cn1_persistent_logins WHERE username_key = ?", new Object[] {key}); - db.execute("DELETE FROM cn1_mfa_totp WHERE username_key = ?", new Object[] {key}); - db.execute("DELETE FROM cn1_mfa_recovery_code WHERE username_key = ?", new Object[] {key}); - db.execute("DELETE FROM cn1_federated_identity WHERE username_key = ?", new Object[] {key}); - db.execute("DELETE FROM cn1_webauthn_credential WHERE user_id IN " - + "(SELECT user_id FROM cn1_webauthn_user WHERE username_key = ?)", new Object[] {key}); - db.execute("DELETE FROM cn1_webauthn_user WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_persistent_logins") + " WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_totp") + " WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_recovery_code") + " WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_federated_identity") + " WHERE username_key = ?", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_webauthn_credential") + " WHERE user_id IN " + + "(SELECT user_id FROM " + SecuritySchema.table("cn1_webauthn_user") + " WHERE username_key = ?)", new Object[] {key}); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_webauthn_user") + " WHERE username_key = ?", new Object[] {key}); // These two stores retain the principal's original spelling rather than // username_key. Use the same ASCII-only fold as the user store, independent // of the database's locale and collation. Client-credentials grants name a // client rather than a user and must survive a user with the same name. - for (Object row : db.query("SELECT key_hash, owner FROM cn1_api_key", null)) { + for (Object row : db.query("SELECT key_hash, owner FROM " + SecuritySchema.table("cn1_api_key"), null)) { Map values = (Map) row; if (key.equals(SecuritySchema.usernameKey((String) values.get("owner")))) { - db.execute("DELETE FROM cn1_api_key WHERE key_hash = ?", + db.execute("DELETE FROM " + SecuritySchema.table("cn1_api_key") + " WHERE key_hash = ?", new Object[] {values.get("key_hash")}); } } - for (Object row : db.query("SELECT id, principal_name FROM cn1_oauth2_authorization " + for (Object row : db.query("SELECT id, principal_name FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " " + "WHERE grant_type <> 'client_credentials'", null)) { Map values = (Map) row; if (key.equals(SecuritySchema.usernameKey((String) values.get("principal_name")))) { Object[] id = {values.get("id")}; - db.execute("DELETE FROM cn1_oauth2_token WHERE authorization_id = ?", id); - db.execute("DELETE FROM cn1_oauth2_authorization WHERE id = ?", id); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE authorization_id = ?", id); + db.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ?", id); } } } @@ -245,7 +245,7 @@ private static void deleteCredentials(Database db, String key) throws java.io.IO @Override public boolean userExists(String username) { try { - return dataSource.queryOne("SELECT username_key FROM cn1_users WHERE username_key = ?", + return dataSource.queryOne("SELECT username_key FROM " + SecuritySchema.table("cn1_users") + " WHERE username_key = ?", new Object[] {SecuritySchema.usernameKey(username)}) != null; } catch (java.io.IOException err) { throw failed("look the user up", err); @@ -268,7 +268,7 @@ public void changePassword(String oldPassword, String newPassword) { private boolean setPassword(String username, String password) { try { - return dataSource.execute("UPDATE cn1_users SET password = ? WHERE username_key = ?", + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_users") + " SET password = ? WHERE username_key = ?", new Object[] {password, SecuritySchema.usernameKey(username)}) == 1; } catch (java.io.IOException err) { throw failed("store the password", err); diff --git a/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java b/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java new file mode 100644 index 00000000000..d4be4a6d85e --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.crypto; + +import com.codename1.backend.Crypto; +import java.io.UnsupportedEncodingException; + +/// The hexadecimal SHA-256 of a value, unsalted and in one round. Registered as +/// `{sha256}`. +/// +/// This exists to verify values another system stored this way, so that they +/// can be imported without asking every user for new ones: recovery codes, or +/// tokens that are long and random. A single unsalted digest is far too fast for +/// a password a person chose; never encode those with it. +/// +/// [#matches(CharSequence, String)] compares in constant time and accepts upper +/// or lower case hexadecimal, and [#upgradeEncoding(String)] is always true. +public final class Sha256HexPasswordEncoder implements PasswordEncoder { + private static final char[] DIGITS = "0123456789abcdef".toCharArray(); + + @Override + public String encode(CharSequence rawPassword) { + if (rawPassword == null) { + throw new IllegalArgumentException("rawPassword cannot be null"); + } + byte[] digest = Crypto.sha256(utf8(rawPassword.toString())); + char[] out = new char[digest.length * 2]; + for (int iter = 0 ; iter < digest.length ; iter++) { + out[iter * 2] = DIGITS[(digest[iter] >> 4) & 0xf]; + out[iter * 2 + 1] = DIGITS[digest[iter] & 0xf]; + } + return new String(out); + } + + @Override + public boolean matches(CharSequence rawPassword, String encodedPassword) { + if (rawPassword == null || encodedPassword == null || encodedPassword.length() != 64) { + return false; + } + String actual = encode(rawPassword); + int diff = 0; + for (int iter = 0 ; iter < 64 ; iter++) { + int c = encodedPassword.charAt(iter); + // Upper case hex folded with arithmetic, so the time does not depend on it. + int upper = ((c - 'A') >>> 31 ^ 1) & (('F' - c) >>> 31 ^ 1); + c += upper * ('a' - 'A'); + diff |= c ^ actual.charAt(iter); + } + return diff == 0; + } + + @Override + public boolean upgradeEncoding(String encodedPassword) { + return true; + } + + private static byte[] utf8(String value) { + try { + return value.getBytes("UTF-8"); + } catch (UnsupportedEncodingException err) { + throw new IllegalStateException("UTF-8 is required", err); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mfa/JdbcRecoveryCodeRepository.java b/vm/backend/src/com/codename1/backend/security/mfa/JdbcRecoveryCodeRepository.java index 73f67be2b2f..f26ec352acc 100644 --- a/vm/backend/src/com/codename1/backend/security/mfa/JdbcRecoveryCodeRepository.java +++ b/vm/backend/src/com/codename1/backend/security/mfa/JdbcRecoveryCodeRepository.java @@ -75,10 +75,10 @@ private static final class Replace implements DataSource.Work { @Override public Object run(Database db) throws Exception { - db.execute("DELETE FROM cn1_mfa_recovery_code WHERE username_key = ?", + db.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_recovery_code") + " WHERE username_key = ?", new Object[] {user}); for (String hash : codeHashes) { - db.execute("INSERT INTO cn1_mfa_recovery_code (username_key, code_hash) " + db.execute("INSERT INTO " + SecuritySchema.table("cn1_mfa_recovery_code") + " (username_key, code_hash) " + "VALUES (?, ?)", new Object[] {user, hash}); } return null; @@ -92,7 +92,7 @@ public List findHashes(String username) { return hashes; } try { - for (Object row : dataSource.query("SELECT code_hash FROM cn1_mfa_recovery_code " + for (Object row : dataSource.query("SELECT code_hash FROM " + SecuritySchema.table("cn1_mfa_recovery_code") + " " + "WHERE username_key = ?", new Object[] {SecuritySchema.usernameKey(username)})) { hashes.add((String) ((Map) row).get("code_hash")); } @@ -108,7 +108,7 @@ public boolean consume(String username, String codeHash) { return false; } try { - return dataSource.execute("DELETE FROM cn1_mfa_recovery_code WHERE username_key = ? " + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_recovery_code") + " WHERE username_key = ? " + "AND code_hash = ?", new Object[] {SecuritySchema.usernameKey(username), codeHash}) == 1; } catch (IOException err) { @@ -119,7 +119,7 @@ public boolean consume(String username, String codeHash) { @Override public int count(String username) { try { - Map row = dataSource.queryOne("SELECT COUNT(*) AS n FROM cn1_mfa_recovery_code WHERE " + Map row = dataSource.queryOne("SELECT COUNT(*) AS n FROM " + SecuritySchema.table("cn1_mfa_recovery_code") + " WHERE " + "username_key = ?", new Object[] {SecuritySchema.usernameKey(username)}); Object n = row == null ? null : row.get("n"); return n instanceof Number ? ((Number) n).intValue() : 0; diff --git a/vm/backend/src/com/codename1/backend/security/mfa/JdbcTotpRepository.java b/vm/backend/src/com/codename1/backend/security/mfa/JdbcTotpRepository.java index 27a3433cc90..435110cd59c 100644 --- a/vm/backend/src/com/codename1/backend/security/mfa/JdbcTotpRepository.java +++ b/vm/backend/src/com/codename1/backend/security/mfa/JdbcTotpRepository.java @@ -155,8 +155,8 @@ private static final class Store implements DataSource.Work { @Override public Object run(Database db) throws Exception { - db.execute("DELETE FROM cn1_mfa_totp WHERE username_key = ?", new Object[] {user}); - db.execute("INSERT INTO cn1_mfa_totp (username_key, secret, nonce, confirmed, " + db.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_totp") + " WHERE username_key = ?", new Object[] {user}); + db.execute("INSERT INTO " + SecuritySchema.table("cn1_mfa_totp") + " (username_key, secret, nonce, confirmed, " + "last_used_step, created_at) VALUES (?, ?, ?, 0, -1, ?)", new Object[] {user, sealed, nonce, Long.valueOf(now)}); return null; @@ -171,7 +171,7 @@ public TotpCredential find(String username) { String user = SecuritySchema.usernameKey(username); try { Map row = dataSource.queryOne("SELECT secret, nonce, confirmed, last_used_step FROM " - + "cn1_mfa_totp WHERE username_key = ?", new Object[] {user}); + + SecuritySchema.table("cn1_mfa_totp") + " WHERE username_key = ?", new Object[] {user}); if (row == null) { return null; } @@ -198,7 +198,7 @@ public TotpCredential find(String username) { @Override public boolean confirm(String username) { try { - return dataSource.execute("UPDATE cn1_mfa_totp SET confirmed = 1 WHERE " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_mfa_totp") + " SET confirmed = 1 WHERE " + "username_key = ? AND confirmed = 0", new Object[] {SecuritySchema.usernameKey(username)}) == 1; } catch (IOException err) { @@ -209,7 +209,7 @@ public boolean confirm(String username) { @Override public boolean advance(String username, long step) { try { - return dataSource.execute("UPDATE cn1_mfa_totp SET last_used_step = ? WHERE " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_mfa_totp") + " SET last_used_step = ? WHERE " + "username_key = ? AND last_used_step < ?", new Object[] {Long.valueOf(step), SecuritySchema.usernameKey(username), Long.valueOf(step)}) == 1; } catch (IOException err) { @@ -233,7 +233,7 @@ private boolean accept(String username, byte[] expectedSecret, long step, boolea } String user = SecuritySchema.usernameKey(username); try { - Map row = dataSource.queryOne("SELECT secret, nonce FROM cn1_mfa_totp " + Map row = dataSource.queryOne("SELECT secret, nonce FROM " + SecuritySchema.table("cn1_mfa_totp") + " " + "WHERE username_key = ?", new Object[] {user}); if (row == null) { return false; @@ -247,7 +247,7 @@ private boolean accept(String username, byte[] expectedSecret, long step, boolea } // A concurrent save changes the sealed secret and nonce. The predicate // binds this update to the row just checked, even on another server. - return dataSource.execute("UPDATE cn1_mfa_totp SET confirmed = 1, last_used_step = ? " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_mfa_totp") + " SET confirmed = 1, last_used_step = ? " + "WHERE username_key = ? AND secret = ? AND nonce = ? AND confirmed = ? " + "AND last_used_step < ?", new Object[] {Long.valueOf(step), user, sealed, nonce, Integer.valueOf(confirming ? 0 : 1), Long.valueOf(step)}) == 1; @@ -259,7 +259,7 @@ private boolean accept(String username, byte[] expectedSecret, long step, boolea @Override public boolean delete(String username) { try { - return dataSource.execute("DELETE FROM cn1_mfa_totp WHERE username_key = ?", + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_mfa_totp") + " WHERE username_key = ?", new Object[] {SecuritySchema.usernameKey(username)}) == 1; } catch (IOException err) { throw failed(err); diff --git a/vm/backend/src/com/codename1/backend/security/mfa/RecoveryCodeService.java b/vm/backend/src/com/codename1/backend/security/mfa/RecoveryCodeService.java index bcb222fdec1..d9edd50d8d5 100644 --- a/vm/backend/src/com/codename1/backend/security/mfa/RecoveryCodeService.java +++ b/vm/backend/src/com/codename1/backend/security/mfa/RecoveryCodeService.java @@ -24,10 +24,14 @@ import com.codename1.backend.Crypto; import com.codename1.backend.security.SecuritySchema; +import com.codename1.backend.security.crypto.DelegatingPasswordEncoder; import com.codename1.backend.security.crypto.Pbkdf2Sha256PasswordEncoder; import com.codename1.backend.security.crypto.PasswordEncoder; +import com.codename1.backend.security.crypto.Sha256HexPasswordEncoder; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import java.util.List; import java.util.HashSet; import java.util.Set; @@ -49,13 +53,49 @@ public final class RecoveryCodeService { private static final int MAX_VERIFYING = 4; private final RecoveryCodeRepository repository; - private final PasswordEncoder encoder = new Pbkdf2Sha256PasswordEncoder(); + private final PasswordEncoder encoder; + /// A service whose codes are hashed by [#defaultEncoder()]. public RecoveryCodeService(RecoveryCodeRepository repository) { + this(repository, defaultEncoder()); + } + + /// A service whose codes are hashed and verified by `encoder`, for codes + /// imported from a system that stored them another way. + /// + /// #### Parameters + /// + /// - `repository`: where the hashes are kept + /// - `encoder`: hashes new codes and verifies stored ones + public RecoveryCodeService(RecoveryCodeRepository repository, PasswordEncoder encoder) { if (repository == null) { throw new IllegalArgumentException("repository cannot be null"); } + if (encoder == null) { + throw new IllegalArgumentException("encoder cannot be null"); + } this.repository = repository; + this.encoder = encoder; + } + + /// The encoder recovery codes use unless another is given: a + /// [DelegatingPasswordEncoder] that hashes new codes as `{pbkdf2-sha256}`, and + /// verifies a stored hash by the `{id}` in front of it -- `{pbkdf2-sha256}`, + /// or `{sha256}` for the hexadecimal SHA-256 of the code another system may + /// have stored ([Sha256HexPasswordEncoder]). A hash with no `{id}` is + /// PBKDF2, as every code this service made before ids were written. + /// + /// A code is compared in the normalized form a user's input is reduced to: + /// lower case, without `-` or spaces. Codes imported as `{sha256}` must be + /// hashed from that form. + public static PasswordEncoder defaultEncoder() { + Map encoders = new HashMap(); + PasswordEncoder pbkdf2 = new Pbkdf2Sha256PasswordEncoder(); + encoders.put("pbkdf2-sha256", pbkdf2); + encoders.put("sha256", new Sha256HexPasswordEncoder()); + DelegatingPasswordEncoder out = new DelegatingPasswordEncoder("pbkdf2-sha256", encoders); + out.setDefaultPasswordEncoderForMatches(pbkdf2); + return out; } /// Makes [#COUNT] new codes for `username`, replacing any they had. diff --git a/vm/backend/src/com/codename1/backend/security/oauth2/client/JdbcFederatedIdentityRepository.java b/vm/backend/src/com/codename1/backend/security/oauth2/client/JdbcFederatedIdentityRepository.java index 4a3faf8f1cd..b9daceddfe6 100644 --- a/vm/backend/src/com/codename1/backend/security/oauth2/client/JdbcFederatedIdentityRepository.java +++ b/vm/backend/src/com/codename1/backend/security/oauth2/client/JdbcFederatedIdentityRepository.java @@ -72,7 +72,7 @@ public String findUsername(String provider, String subject) { } private String read(String provider, String subject) throws IOException { - Map row = dataSource.queryOne("SELECT username FROM cn1_federated_identity WHERE " + Map row = dataSource.queryOne("SELECT username FROM " + SecuritySchema.table("cn1_federated_identity") + " WHERE " + "provider = ? AND subject = ?", new Object[] {provider, subject}); return row == null ? null : String.valueOf(row.get("username")); } @@ -85,7 +85,7 @@ public String link(String provider, String subject, String username) { return existing; } try { - dataSource.execute("INSERT INTO cn1_federated_identity (provider, subject, " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_federated_identity") + " (provider, subject, " + "username_key, username, created_at) VALUES (?, ?, ?, ?, ?)", new Object[] {provider, subject, SecuritySchema.usernameKey(username), username, Long.valueOf(clock.currentTimeMillis())}); @@ -108,7 +108,7 @@ public String link(String provider, String subject, String username) { @Override public boolean unlink(String provider, String subject) { try { - return dataSource.execute("DELETE FROM cn1_federated_identity WHERE provider = ? " + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_federated_identity") + " WHERE provider = ? " + "AND subject = ?", new Object[] {provider, subject}) == 1; } catch (IOException err) { throw new IllegalStateException("The federated identity could not be removed: " @@ -121,7 +121,7 @@ public List findByUsername(String username) { List out = new ArrayList(); try { for (Object row : dataSource.query("SELECT provider, subject FROM " - + "cn1_federated_identity WHERE username_key = ? ORDER BY created_at, " + + SecuritySchema.table("cn1_federated_identity") + " WHERE username_key = ? ORDER BY created_at, " + "provider, subject", new Object[] {SecuritySchema.usernameKey(username)})) { Map r = (Map) row; out.add(new String[] {String.valueOf(r.get("provider")), diff --git a/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcOAuth2AuthorizationService.java b/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcOAuth2AuthorizationService.java index 3d774c5f0b7..352efe2975e 100644 --- a/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcOAuth2AuthorizationService.java +++ b/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcOAuth2AuthorizationService.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security.oauth2.server.authorization; +import com.codename1.backend.security.SecuritySchema; import com.codename1.backend.DataSource; import com.codename1.backend.Database; import com.codename1.backend.Json; @@ -77,12 +78,12 @@ public void save(OAuth2Authorization a) { try { // An update that changes nothing counts no row on MySQL, so "no row" // alone does not say the grant is new: it is asked for. - if (dataSource.execute("UPDATE cn1_oauth2_authorization SET registered_client_id = ?, " + if (dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_authorization") + " SET registered_client_id = ?, " + "principal_name = ?, grant_type = ?, scopes = ?, status = ?, " + "attributes = ?, expires_at = ? WHERE id = ?", values) == 0 - && dataSource.queryOne("SELECT id FROM cn1_oauth2_authorization WHERE id = ?", + && dataSource.queryOne("SELECT id FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ?", new Object[] {a.getId()}) == null) { - dataSource.execute("INSERT INTO cn1_oauth2_authorization (registered_client_id, " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_oauth2_authorization") + " (registered_client_id, " + "principal_name, grant_type, scopes, status, attributes, expires_at, " + "id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", new Object[] { values[0], values[1], values[2], values[3], values[4], values[5], @@ -101,7 +102,7 @@ public OAuth2Authorization findById(String id) { try { Map row = dataSource.queryOne("SELECT id, registered_client_id, principal_name, " + "grant_type, scopes, status, attributes, created_at, expires_at FROM " - + "cn1_oauth2_authorization WHERE id = ?", new Object[] {id}); + + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ?", new Object[] {id}); if (row == null) { return null; } @@ -132,9 +133,9 @@ public void remove(String id) { try { // The grant first: a secret whose grant is gone is worth nothing, // whichever of these another process sees half done. - dataSource.execute("DELETE FROM cn1_oauth2_authorization WHERE id = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ?", new Object[] {id}); - dataSource.execute("DELETE FROM cn1_oauth2_token WHERE authorization_id = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE authorization_id = ?", new Object[] {id}); } catch (IOException err) { throw failed("remove a grant", err); @@ -144,7 +145,7 @@ public void remove(String id) { @Override public void addToken(String authorizationId, String kind, String tokenHash, long expiresAt) { try { - dataSource.execute("INSERT INTO cn1_oauth2_token (token_hash, authorization_id, " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_oauth2_token") + " (token_hash, authorization_id, " + "kind, used, expires_at, polled_at) VALUES (?, ?, ?, 0, ?, 0)", new Object[] {tokenHash, authorizationId, kind, Long.valueOf(expiresAt)}); } catch (IOException err) { @@ -186,32 +187,32 @@ public Object run(Database db) throws Exception { Object[] grant = {id, OAuth2Authorization.ACTIVE, Long.valueOf(now)}; // A portable row lock, including SQLite. MySQL may report zero for // this unchanged value, so existence is checked while holding it. - db.execute("UPDATE cn1_oauth2_authorization SET expires_at = expires_at " + db.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_authorization") + " SET expires_at = expires_at " + "WHERE id = ? AND status = ? AND expires_at > ?", grant); - if (db.queryOne("SELECT id FROM cn1_oauth2_authorization WHERE id = ? " + if (db.queryOne("SELECT id FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ? " + "AND status = ? AND expires_at > ?", grant) == null) { return Boolean.FALSE; } if (hash != null) { if (reuse) { Object[] token = {hash, id, REFRESH_TOKEN, Long.valueOf(now)}; - db.execute("UPDATE cn1_oauth2_token SET expires_at = CASE WHEN expires_at < ? " + db.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_token") + " SET expires_at = CASE WHEN expires_at < ? " + "THEN ? ELSE expires_at END WHERE token_hash = ? AND authorization_id = ? " + "AND kind = ? AND used = 0 AND expires_at > ?", new Object[] { Long.valueOf(expiresAt), Long.valueOf(expiresAt), hash, id, REFRESH_TOKEN, Long.valueOf(now)}); - if (db.queryOne("SELECT token_hash FROM cn1_oauth2_token WHERE token_hash = ? " + if (db.queryOne("SELECT token_hash FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE token_hash = ? " + "AND authorization_id = ? AND kind = ? AND used = 0 AND expires_at > ?", token) == null) { return Boolean.FALSE; } } else { - db.execute("INSERT INTO cn1_oauth2_token (token_hash, authorization_id, " + db.execute("INSERT INTO " + SecuritySchema.table("cn1_oauth2_token") + " (token_hash, authorization_id, " + "kind, used, expires_at, polled_at) VALUES (?, ?, ?, 0, ?, 0)", new Object[] {hash, id, REFRESH_TOKEN, Long.valueOf(expiresAt)}); } } - db.execute("UPDATE cn1_oauth2_authorization SET expires_at = CASE WHEN expires_at < ? " + db.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_authorization") + " SET expires_at = CASE WHEN expires_at < ? " + "THEN ? ELSE expires_at END WHERE id = ?", new Object[] {Long.valueOf(expiresAt), Long.valueOf(expiresAt), id}); return Boolean.TRUE; @@ -225,7 +226,7 @@ public StoredToken findToken(String kind, String tokenHash) { } try { Map row = dataSource.queryOne("SELECT authorization_id, used, expires_at, polled_at " - + "FROM cn1_oauth2_token WHERE token_hash = ? AND kind = ?", + + "FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE token_hash = ? AND kind = ?", new Object[] {tokenHash, kind}); return row == null ? null : new StoredToken(String.valueOf(row.get("authorization_id")), number(row.get("used")) != 0, number(row.get("expires_at")), @@ -241,7 +242,7 @@ public boolean consumeToken(String kind, String tokenHash, long now) { return false; } try { - return dataSource.execute("UPDATE cn1_oauth2_token SET used = 1, polled_at = ? " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_token") + " SET used = 1, polled_at = ? " + "WHERE token_hash = ? AND kind = ? AND used = 0 AND expires_at > ?", new Object[] {Long.valueOf(now), tokenHash, kind, Long.valueOf(now)}) == 1; } catch (IOException err) { @@ -252,7 +253,7 @@ public boolean consumeToken(String kind, String tokenHash, long now) { @Override public void touchToken(String kind, String tokenHash, long now) { try { - dataSource.execute("UPDATE cn1_oauth2_token SET polled_at = ? WHERE token_hash = ? " + dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_token") + " SET polled_at = ? WHERE token_hash = ? " + "AND kind = ?", new Object[] {Long.valueOf(now), tokenHash, kind}); } catch (IOException err) { throw failed("record a poll", err); @@ -266,7 +267,7 @@ public boolean pollToken(String kind, String tokenHash, long now, long intervalM } long before = now < intervalMillis ? 0 : now - intervalMillis; try { - return dataSource.execute("UPDATE cn1_oauth2_token SET polled_at = ? WHERE token_hash = ? " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_token") + " SET polled_at = ? WHERE token_hash = ? " + "AND kind = ? AND used = 0 AND expires_at > ? " + "AND (polled_at = 0 OR polled_at <= ?)", new Object[] {Long.valueOf(now), tokenHash, kind, Long.valueOf(now), @@ -279,7 +280,7 @@ public boolean pollToken(String kind, String tokenHash, long now, long intervalM @Override public boolean extendToken(String kind, String tokenHash, long now, long expiresAt) { try { - return dataSource.execute("UPDATE cn1_oauth2_token SET expires_at = CASE WHEN expires_at < ? " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_token") + " SET expires_at = CASE WHEN expires_at < ? " + "THEN ? ELSE expires_at END WHERE token_hash = ? AND kind = ? AND used = 0 " + "AND expires_at > ?", new Object[] {Long.valueOf(expiresAt), Long.valueOf(expiresAt), tokenHash, kind, Long.valueOf(now)}) == 1; @@ -303,7 +304,7 @@ public boolean decide(String id, boolean approved, String principalName, // The status in the condition is what makes one answer win; the // attributes written are this caller's reading with its own added, // and nothing else writes a pending grant's. - return dataSource.execute("UPDATE cn1_oauth2_authorization SET status = ?, " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_authorization") + " SET status = ?, " + "principal_name = ?, attributes = ? WHERE id = ? AND status = ?", new Object[] {approved ? OAuth2Authorization.ACTIVE : OAuth2Authorization.DENIED, approved ? principalName : "", @@ -320,23 +321,23 @@ public int purgeExpired(long now, int limit) { try { // Found by the index on expires_at, a bounded number at a time, and // deleted by key: a purge never holds more than one row. - List tokens = dataSource.query("SELECT token_hash FROM cn1_oauth2_token WHERE " + List tokens = dataSource.query("SELECT token_hash FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE " + "expires_at < ? ORDER BY expires_at LIMIT " + bound, new Object[] {Long.valueOf(now)}); for (Object row : tokens) { - purged += dataSource.execute("DELETE FROM cn1_oauth2_token WHERE token_hash = ? " + purged += dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE token_hash = ? " + "AND expires_at < ?", new Object[] {((Map) row).get("token_hash"), Long.valueOf(now)}); } - List grants = dataSource.query("SELECT id FROM cn1_oauth2_authorization WHERE " + List grants = dataSource.query("SELECT id FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE " + "expires_at < ? ORDER BY expires_at LIMIT " + bound, new Object[] {Long.valueOf(now)}); for (Object row : grants) { Object id = ((Map) row).get("id"); // Conditional again: a refresh may have moved the expiry since. - if (dataSource.execute("DELETE FROM cn1_oauth2_authorization WHERE id = ? AND " + if (dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_authorization") + " WHERE id = ? AND " + "expires_at < ?", new Object[] {id, Long.valueOf(now)}) == 1) { - dataSource.execute("DELETE FROM cn1_oauth2_token WHERE authorization_id = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_oauth2_token") + " WHERE authorization_id = ?", new Object[] {id}); purged++; } diff --git a/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcRegisteredClientRepository.java b/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcRegisteredClientRepository.java index 72d85575ec7..9e64396c33b 100644 --- a/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcRegisteredClientRepository.java +++ b/vm/backend/src/com/codename1/backend/security/oauth2/server/authorization/JdbcRegisteredClientRepository.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security.oauth2.server.authorization; +import com.codename1.backend.security.SecuritySchema; import com.codename1.backend.DataSource; import com.codename1.backend.Json; import com.codename1.backend.security.Clock; @@ -94,11 +95,11 @@ public void save(RegisteredClient client) { Object[] update = new Object[values.length + 1]; System.arraycopy(values, 0, update, 0, values.length); update[values.length] = client.getId(); - if (dataSource.execute("UPDATE cn1_oauth2_registered_client SET client_id = ?, " + if (dataSource.execute("UPDATE " + SecuritySchema.table("cn1_oauth2_registered_client") + " SET client_id = ?, " + "client_secret = ?, client_name = ?, authentication_methods = ?, " + "grant_types = ?, redirect_uris = ?, scopes = ?, settings = ? WHERE id = ?", update) == 0 && dataSource.queryOne("SELECT id FROM " - + "cn1_oauth2_registered_client WHERE id = ?", + + SecuritySchema.table("cn1_oauth2_registered_client") + " WHERE id = ?", new Object[] {client.getId()}) == null) { // An update that changes nothing counts no row on MySQL, so the // client is asked for before it is taken to be new. @@ -106,7 +107,7 @@ public void save(RegisteredClient client) { insert[0] = client.getId(); System.arraycopy(values, 0, insert, 1, values.length); insert[values.length + 1] = Long.valueOf(clock.currentTimeMillis()); - dataSource.execute("INSERT INTO cn1_oauth2_registered_client (" + COLUMNS + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_oauth2_registered_client") + " (" + COLUMNS + ", created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", insert); } } catch (IOException err) { @@ -141,7 +142,7 @@ private RegisteredClient find(String column, String value) { } Map row; try { - row = dataSource.queryOne("SELECT " + COLUMNS + " FROM cn1_oauth2_registered_client " + row = dataSource.queryOne("SELECT " + COLUMNS + " FROM " + SecuritySchema.table("cn1_oauth2_registered_client") + " " + "WHERE " + column + " = ?", new Object[] {value}); } catch (IOException err) { // A store that cannot be asked knows no client: the request is diff --git a/vm/backend/src/com/codename1/backend/security/ratelimit/JdbcRateLimiter.java b/vm/backend/src/com/codename1/backend/security/ratelimit/JdbcRateLimiter.java index 782f16c1272..4eccec87580 100644 --- a/vm/backend/src/com/codename1/backend/security/ratelimit/JdbcRateLimiter.java +++ b/vm/backend/src/com/codename1/backend/security/ratelimit/JdbcRateLimiter.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security.ratelimit; +import com.codename1.backend.security.SecuritySchema; import com.codename1.backend.Base64Url; import com.codename1.backend.Crypto; import com.codename1.backend.DataSource; @@ -129,20 +130,20 @@ public boolean tryAcquire(String key) { // The window has passed: start the next one. Only one of several // callers at this moment changes the row; the rest count in the // window it started. - if (dataSource.execute("UPDATE cn1_rate_limit SET window_start = ?, window_end = ?, hits = 1 WHERE " + if (dataSource.execute("UPDATE " + SecuritySchema.table("cn1_rate_limit") + " SET window_start = ?, window_end = ?, hits = 1 WHERE " + "limit_key = ? AND window_start <= ?", new Object[] {Long.valueOf(now), Long.valueOf(windowEnd(now)), row, windowFloor}) == 1) { return true; } - if (dataSource.queryOne("SELECT hits FROM cn1_rate_limit WHERE limit_key = ?", + if (dataSource.queryOne("SELECT hits FROM " + SecuritySchema.table("cn1_rate_limit") + " WHERE limit_key = ?", new Object[] {row}) != null) { // The row is there and neither statement took it: the window is // full, or another caller has just started the next one. return count(row, windowFloor); } try { - dataSource.execute("INSERT INTO cn1_rate_limit (limit_key, window_start, window_end, hits) " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_rate_limit") + " (limit_key, window_start, window_end, hits) " + "VALUES (?, ?, ?, 1)", new Object[] {row, Long.valueOf(now), Long.valueOf(windowEnd(now))}); return true; } catch (IOException raced) { @@ -164,7 +165,7 @@ public void reset(String key) { return; } try { - dataSource.execute("DELETE FROM cn1_rate_limit WHERE limit_key = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_rate_limit") + " WHERE limit_key = ?", new Object[] {row(key)}); } catch (IOException err) { System.err.println("cn1: the rate limit store could not be reached, so a count " @@ -175,7 +176,7 @@ public void reset(String key) { /// One more hit in the row's window, when that window is still open and /// has room. private boolean count(String row, Long windowFloor) throws IOException { - return dataSource.execute("UPDATE cn1_rate_limit SET hits = hits + 1 WHERE limit_key = ? " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_rate_limit") + " SET hits = hits + 1 WHERE limit_key = ? " + "AND window_start > ? AND hits < ?", new Object[] {row, windowFloor, Integer.valueOf(permits)}) == 1; } @@ -186,7 +187,7 @@ public long retryAfterSeconds(String key) { return 1; } try { - java.util.Map found = dataSource.queryOne("SELECT window_start FROM cn1_rate_limit " + java.util.Map found = dataSource.queryOne("SELECT window_start FROM " + SecuritySchema.table("cn1_rate_limit") + " " + "WHERE limit_key = ?", new Object[] {row(key)}); Object start = found == null ? null : found.get("window_start"); if (!(start instanceof Number)) { @@ -212,7 +213,7 @@ public int deleteExpired(long olderThanSeconds) throws IOException { long retention = olderThanSeconds * 1000L; long now = clock.currentTimeMillis(); long cutoff = now < Long.MIN_VALUE + retention ? Long.MIN_VALUE : now - retention; - return dataSource.execute("DELETE FROM cn1_rate_limit WHERE window_end < ?", + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_rate_limit") + " WHERE window_end < ?", new Object[] {Long.valueOf(cutoff)}); } diff --git a/vm/backend/src/com/codename1/backend/security/rememberme/InMemoryTokenRepositoryImpl.java b/vm/backend/src/com/codename1/backend/security/rememberme/InMemoryTokenRepositoryImpl.java index 5239b5b6da3..d2c73a4a499 100644 --- a/vm/backend/src/com/codename1/backend/security/rememberme/InMemoryTokenRepositoryImpl.java +++ b/vm/backend/src/com/codename1/backend/security/rememberme/InMemoryTokenRepositoryImpl.java @@ -22,8 +22,11 @@ */ package com.codename1.backend.security.rememberme; +import com.codename1.backend.security.SecuritySchema; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; /// Remembered sign-ins kept in this process: gone when it stops, and unknown to @@ -50,7 +53,8 @@ public synchronized boolean updateToken(String series, String expectedTokenHash, return false; } bySeries.put(series, new PersistentRememberMeToken(current.getUsername(), series, - newTokenHash, lastUsed, expectedTokenHash)); + newTokenHash, lastUsed, expectedTokenHash, current.getUserAgent(), + current.getIpAddress())); return true; } @@ -75,6 +79,23 @@ public synchronized void removeUserTokens(String username) { } /// How many series are stored. + @Override + public synchronized List getTokensForUser(String username) { + List out = new ArrayList(); + String key = SecuritySchema.usernameKey(username); + for (PersistentRememberMeToken token : bySeries.values()) { + if (key != null && key.equals(SecuritySchema.usernameKey(token.getUsername()))) { + out.add(token); + } + } + return out; + } + + @Override + public synchronized boolean removeUserToken(String username, String series) { + return PersistentTokenRepository.super.removeUserToken(username, series); + } + public synchronized int size() { return bySeries.size(); } diff --git a/vm/backend/src/com/codename1/backend/security/rememberme/JdbcTokenRepository.java b/vm/backend/src/com/codename1/backend/security/rememberme/JdbcTokenRepository.java index a4fa1e05ca1..9b9b504103c 100644 --- a/vm/backend/src/com/codename1/backend/security/rememberme/JdbcTokenRepository.java +++ b/vm/backend/src/com/codename1/backend/security/rememberme/JdbcTokenRepository.java @@ -26,6 +26,8 @@ import com.codename1.backend.security.AuthenticationServiceException; import com.codename1.backend.security.SecuritySchema; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; /// Remembered sign-ins kept in the server's database, in the @@ -56,11 +58,13 @@ private static RuntimeException failed(IOException err) { @Override public void createNewToken(PersistentRememberMeToken token) { try { - dataSource.execute("INSERT INTO cn1_persistent_logins (series, username_key, " - + "username, token_hash, last_used, previous_token_hash) VALUES (?, ?, ?, ?, ?, ?)", new Object[] { + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_persistent_logins") + " (series, username_key, " + + "username, token_hash, last_used, previous_token_hash, user_agent, ip_address) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", new Object[] { token.getSeries(), SecuritySchema.usernameKey(token.getUsername()), token.getUsername(), token.getTokenHash(), - Long.valueOf(token.getLastUsed()), token.getPreviousTokenHash()}); + Long.valueOf(token.getLastUsed()), token.getPreviousTokenHash(), + token.getUserAgent(), token.getIpAddress()}); } catch (IOException err) { throw failed(err); } @@ -70,7 +74,7 @@ public void createNewToken(PersistentRememberMeToken token) { public boolean updateToken(String series, String expectedTokenHash, String newTokenHash, long lastUsed) { try { - return dataSource.execute("UPDATE cn1_persistent_logins SET previous_token_hash = ?, " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_persistent_logins") + " SET previous_token_hash = ?, " + "token_hash = ?, last_used = ? WHERE series = ? AND token_hash = ?", new Object[] { expectedTokenHash, newTokenHash, Long.valueOf(lastUsed), series, expectedTokenHash}) == 1; } catch (IOException err) { @@ -84,16 +88,13 @@ public PersistentRememberMeToken getTokenForSeries(String series) { return null; } try { - Map row = dataSource.queryOne("SELECT username, token_hash, last_used, previous_token_hash FROM " - + "cn1_persistent_logins WHERE series = ?", new Object[] {series}); + Map row = dataSource.queryOne("SELECT username, token_hash, last_used, previous_token_hash, user_agent, " + + "ip_address FROM " + + SecuritySchema.table("cn1_persistent_logins") + " WHERE series = ?", new Object[] {series}); if (row == null) { return null; } - Object used = row.get("last_used"); - return new PersistentRememberMeToken(String.valueOf(row.get("username")), series, - String.valueOf(row.get("token_hash")), - used instanceof Number ? ((Number) used).longValue() : 0L, - (String) row.get("previous_token_hash")); + return token(row, series); } catch (IOException err) { throw failed(err); } @@ -102,7 +103,7 @@ public PersistentRememberMeToken getTokenForSeries(String series) { @Override public void removeToken(String series) { try { - dataSource.execute("DELETE FROM cn1_persistent_logins WHERE series = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_persistent_logins") + " WHERE series = ?", new Object[] {series}); } catch (IOException err) { throw failed(err); @@ -112,7 +113,7 @@ public void removeToken(String series) { @Override public void removeUserTokens(String username) { try { - dataSource.execute("DELETE FROM cn1_persistent_logins WHERE username_key = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_persistent_logins") + " WHERE username_key = ?", new Object[] {SecuritySchema.usernameKey(username)}); } catch (IOException err) { throw failed(err); @@ -124,8 +125,58 @@ public void removeUserTokens(String username) { /// /// @param now epoch milliseconds /// @return how many were deleted + private static PersistentRememberMeToken token(Map row, String series) { + Object used = row.get("last_used"); + return new PersistentRememberMeToken(String.valueOf(row.get("username")), series, + String.valueOf(row.get("token_hash")), + used instanceof Number ? ((Number) used).longValue() : 0L, + text(row.get("previous_token_hash")), text(row.get("user_agent")), + text(row.get("ip_address"))); + } + + private static String text(Object value) { + return value instanceof String ? (String) value : null; + } + + /// Every series of a user, most recently used first. + @Override + public List getTokensForUser(String username) { + List out = new ArrayList(); + if (username == null) { + return out; + } + try { + for (Object row : dataSource.query("SELECT series, username, token_hash, last_used, " + + "previous_token_hash, user_agent, ip_address FROM " + + SecuritySchema.table("cn1_persistent_logins") + " WHERE username_key = ? " + + "ORDER BY last_used DESC", new Object[] {SecuritySchema.usernameKey(username)})) { + Map map = (Map) row; + out.add(token(map, String.valueOf(map.get("series")))); + } + return out; + } catch (IOException err) { + throw failed(err); + } + } + + /// Removes the series in one statement that names its owner, so it cannot + /// remove another user's. + @Override + public boolean removeUserToken(String username, String series) { + if (username == null || series == null) { + return false; + } + try { + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_persistent_logins") + + " WHERE series = ? AND username_key = ?", new Object[] {series, + SecuritySchema.usernameKey(username)}) == 1; + } catch (IOException err) { + throw failed(err); + } + } + public int deleteExpired(long now, long olderThanSeconds) throws IOException { - return dataSource.execute("DELETE FROM cn1_persistent_logins WHERE last_used < ?", + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_persistent_logins") + " WHERE last_used < ?", new Object[] {Long.valueOf(now - olderThanSeconds * 1000L)}); } } diff --git a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentRememberMeToken.java b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentRememberMeToken.java index ee2eef2d997..9cf285b41c1 100644 --- a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentRememberMeToken.java +++ b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentRememberMeToken.java @@ -31,6 +31,8 @@ public final class PersistentRememberMeToken { private final String tokenHash; private final long lastUsed; private final String previousTokenHash; + private final String userAgent; + private final String ipAddress; /// @param tokenHash the SHA-256 of the token, in hex: the token itself is /// only ever in the cookie @@ -51,6 +53,29 @@ public PersistentRememberMeToken(String username, String series, String tokenHas this.tokenHash = tokenHash; this.lastUsed = lastUsed; this.previousTokenHash = previousTokenHash; + this.userAgent = null; + this.ipAddress = null; + } + + /// A token with the device it was issued to, for a "signed-in devices" page. + /// + /// #### Parameters + /// + /// - `userAgent`: the `User-Agent` of the sign-in, or null + /// - `ipAddress`: the client address of the sign-in, or null + public PersistentRememberMeToken(String username, String series, String tokenHash, + long lastUsed, String previousTokenHash, String userAgent, + String ipAddress) { + if (username == null || series == null || tokenHash == null) { + throw new IllegalArgumentException("A token needs a user, a series and a hash"); + } + this.username = username; + this.series = series; + this.tokenHash = tokenHash; + this.lastUsed = lastUsed; + this.previousTokenHash = previousTokenHash; + this.userAgent = userAgent; + this.ipAddress = ipAddress; } public String getUsername() { @@ -73,4 +98,16 @@ public long getLastUsed() { public String getPreviousTokenHash() { return previousTokenHash; } + + /// The `User-Agent` the series was issued to, or null when it was not recorded. + public String getUserAgent() { + return userAgent; + } + + /// The client address the series was issued to, or null when it was not + /// recorded. It is the address at sign-in; the series may since be used from + /// another. + public String getIpAddress() { + return ipAddress; + } } diff --git a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenBasedRememberMeServices.java b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenBasedRememberMeServices.java index cd9333fd2a1..5b0c4aae23c 100644 --- a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenBasedRememberMeServices.java +++ b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenBasedRememberMeServices.java @@ -271,10 +271,19 @@ public void loginSuccess(HttpServer.Request request, Authentication successfulAu String token = random(); tokenRepository.createNewToken(new PersistentRememberMeToken( successfulAuthentication.getName(), series, hash(token), - clock.currentTimeMillis())); + clock.currentTimeMillis(), null, device(request.getHeader("User-Agent")), + device(request.getRemoteAddress()))); setCookie(series + ":" + token); } + // Kept for display only, and bounded: a header is the client's to choose. + private static String device(String value) { + if (value == null) { + return null; + } + return value.length() > 255 ? value.substring(0, 255) : value; + } + @Override public void loginFail(HttpServer.Request request) { String cookie = request.getCookie(cookieName); diff --git a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenRepository.java b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenRepository.java index a79f45185fc..5dc231f6370 100644 --- a/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenRepository.java +++ b/vm/backend/src/com/codename1/backend/security/rememberme/PersistentTokenRepository.java @@ -22,6 +22,9 @@ */ package com.codename1.backend.security.rememberme; +import com.codename1.backend.security.SecuritySchema; +import java.util.List; + /// Where remembered sign-ins are kept. public interface PersistentTokenRepository { /// Stores a new series. @@ -47,4 +50,33 @@ boolean updateToken(String series, String expectedTokenHash, String newTokenHash /// Forgets every series of a user: every browser they were remembered in. void removeUserTokens(String username); + + /// Every series a user holds, for a "signed-in devices" page: one per browser + /// or device that signed in with remember-me and has not signed out. A + /// repository that cannot list throws `UnsupportedOperationException`. + /// + /// #### Parameters + /// + /// - `username`: the user, matched ignoring case as sign-in does + default List getTokensForUser(String username) { + throw new UnsupportedOperationException(getClass().getName() + + " cannot list a user's remember-me tokens"); + } + + /// Revokes one series of a user, such as a device chosen on a "signed-in + /// devices" page. A series of another user is left alone, so a series taken + /// from a request cannot revoke someone else's sign-in. + /// + /// #### Returns + /// + /// whether the series belonged to the user and was removed + default boolean removeUserToken(String username, String series) { + PersistentRememberMeToken token = getTokenForSeries(series); + if (token == null || username == null || !SecuritySchema.usernameKey(username).equals( + SecuritySchema.usernameKey(token.getUsername()))) { + return false; + } + removeToken(series); + return true; + } } diff --git a/vm/backend/src/com/codename1/backend/security/webauthn/JdbcPublicKeyCredentialUserEntityRepository.java b/vm/backend/src/com/codename1/backend/security/webauthn/JdbcPublicKeyCredentialUserEntityRepository.java index bb5bc0d8200..32d90b65d26 100644 --- a/vm/backend/src/com/codename1/backend/security/webauthn/JdbcPublicKeyCredentialUserEntityRepository.java +++ b/vm/backend/src/com/codename1/backend/security/webauthn/JdbcPublicKeyCredentialUserEntityRepository.java @@ -76,7 +76,7 @@ private static PublicKeyCredentialUserEntity entity(Map row) throws IOException private PublicKeyCredentialUserEntity byName(String username) throws IOException { return entity(dataSource.queryOne("SELECT username, user_id, display_name FROM " - + "cn1_webauthn_user WHERE username_key = ?", + + SecuritySchema.table("cn1_webauthn_user") + " WHERE username_key = ?", new Object[] {SecuritySchema.usernameKey(username)})); } @@ -87,7 +87,7 @@ public PublicKeyCredentialUserEntity findById(byte[] id) { } try { return entity(dataSource.queryOne("SELECT username, user_id, display_name FROM " - + "cn1_webauthn_user WHERE user_id = ?", + + SecuritySchema.table("cn1_webauthn_user") + " WHERE user_id = ?", new Object[] {Base64Url.encode(id)})); } catch (IOException err) { throw failed(err); @@ -114,7 +114,7 @@ public PublicKeyCredentialUserEntity save(PublicKeyCredentialUserEntity user) { return first; } try { - dataSource.execute("INSERT INTO cn1_webauthn_user (username_key, username, " + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_webauthn_user") + " (username_key, username, " + "user_id, display_name, created_at) VALUES (?, ?, ?, ?, ?)", new Object[] {SecuritySchema.usernameKey(user.getName()), user.getName(), Base64Url.encode(user.getId()), user.getDisplayName(), @@ -140,7 +140,7 @@ public void delete(byte[] id) { return; } try { - dataSource.execute("DELETE FROM cn1_webauthn_user WHERE user_id = ?", + dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_webauthn_user") + " WHERE user_id = ?", new Object[] {Base64Url.encode(id)}); } catch (IOException err) { throw failed(err); diff --git a/vm/backend/src/com/codename1/backend/security/webauthn/JdbcUserCredentialRepository.java b/vm/backend/src/com/codename1/backend/security/webauthn/JdbcUserCredentialRepository.java index 37fa464590e..bc95a8dfde1 100644 --- a/vm/backend/src/com/codename1/backend/security/webauthn/JdbcUserCredentialRepository.java +++ b/vm/backend/src/com/codename1/backend/security/webauthn/JdbcUserCredentialRepository.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security.webauthn; +import com.codename1.backend.security.SecuritySchema; import com.codename1.backend.Base64Url; import com.codename1.backend.Crypto; import com.codename1.backend.DataSource; @@ -85,7 +86,7 @@ public boolean save(CredentialRecord record) { flag(record.isBackupState()), transports.toString(), record.getLabel(), Long.valueOf(record.getCreated()), Long.valueOf(record.getLastUsed())}; try { - dataSource.execute("INSERT INTO cn1_webauthn_credential (credential_key, " + COLUMNS + dataSource.execute("INSERT INTO " + SecuritySchema.table("cn1_webauthn_credential") + " (credential_key, " + COLUMNS + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", values); return true; } catch (IOException refused) { @@ -99,7 +100,7 @@ public boolean save(CredentialRecord record) { } private CredentialRecord read(byte[] credentialId) throws IOException { - Map row = dataSource.queryOne("SELECT " + COLUMNS + " FROM cn1_webauthn_credential " + Map row = dataSource.queryOne("SELECT " + COLUMNS + " FROM " + SecuritySchema.table("cn1_webauthn_credential") + " " + "WHERE credential_key = ?", new Object[] {key(credentialId)}); return row == null ? null : record(row); } @@ -124,7 +125,7 @@ public List findByUserId(byte[] userEntityUserId) { } try { for (Object row : dataSource.query("SELECT " + COLUMNS + " FROM " - + "cn1_webauthn_credential WHERE user_id = ? ORDER BY created_at, " + + SecuritySchema.table("cn1_webauthn_credential") + " WHERE user_id = ? ORDER BY created_at, " + "credential_key", new Object[] {Base64Url.encode(userEntityUserId)})) { if (row instanceof Map) { out.add(record((Map) row)); @@ -153,7 +154,7 @@ public boolean advance(byte[] credentialId, long signatureCount, boolean uvIniti Long.valueOf(lastUsed), key(credentialId)} : new Object[] {count, flag(uvInitialized), flag(backupState), Long.valueOf(lastUsed), key(credentialId), count}; - return dataSource.execute("UPDATE cn1_webauthn_credential SET sign_count = ?, " + return dataSource.execute("UPDATE " + SecuritySchema.table("cn1_webauthn_credential") + " SET sign_count = ?, " + "uv_initialized = ?, backup_state = ?, last_used = ? WHERE " + "credential_key = ? AND " + moved, values) == 1; } catch (IOException err) { @@ -167,7 +168,7 @@ public boolean delete(byte[] credentialId) { return false; } try { - return dataSource.execute("DELETE FROM cn1_webauthn_credential WHERE " + return dataSource.execute("DELETE FROM " + SecuritySchema.table("cn1_webauthn_credential") + " WHERE " + "credential_key = ?", new Object[] {key(credentialId)}) == 1; } catch (IOException err) { throw failed(err); From 400f32d39d0861858642d8e849908e60a345877a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:34:02 +0200 Subject: [PATCH 09/25] Backend: clear the static analysis findings and follow the security schema to version 12 Co-Authored-By: Claude Opus 5.5 --- .../maven/processors/MigrationGeneratorTest.java | 10 ++++++---- .../javase/com/codename1/backend/JdkHttp2.java | 16 ++++++++-------- .../src/com/codename1/backend/Backend.java | 6 +++--- vm/backend/src/com/codename1/backend/Cors.java | 4 ++-- vm/backend/src/com/codename1/backend/Json.java | 6 +++--- .../src/com/codename1/backend/log/Level.java | 7 +++---- .../src/com/codename1/backend/log/MDC.java | 4 +--- 7 files changed, 26 insertions(+), 27 deletions(-) diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java index d1255567215..cc0e2dd2c35 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java @@ -338,7 +338,7 @@ public void theGoalsApplyTheSecurityTablesAModuleAskedFor() throws Exception { try { assertEquals(Long.valueOf(0), pool.queryOne("SELECT COUNT(*) AS n FROM cn1_users", null) .get("n")); - assertEquals(Long.valueOf(11), pool.queryOne( + assertEquals(Long.valueOf(12), pool.queryOne( "SELECT COUNT(*) AS n FROM cn1_security_schema_history", null).get("n")); assertEquals(Long.valueOf(1), pool.queryOne("SELECT COUNT(*) AS n FROM profiles", null) .get("n")); @@ -400,15 +400,17 @@ public void aModuleWhoseOnlyMigrationsAreTheSecurityTablesGetsTheGoals() throws names.add(line.substring("cn1: security: applied ".length()).trim()); } } - assertEquals(migrated, 11, names.size()); + assertEquals(migrated, 12, names.size()); assertTrue(migrated, names.contains( "com.codename1.backend.security.SecuritySchema.V5__rate_limits")); assertTrue(migrated, names.contains( "com.codename1.backend.security.SecuritySchema.V11__remember_me_rotation_grace")); - assertTrue(migrated, migrated.contains("cn1: security: 11 applied, schema at version 11")); + assertTrue(migrated, names.contains( + "com.codename1.backend.security.SecuritySchema.V12__remember_me_devices")); + assertTrue(migrated, migrated.contains("cn1: security: 12 applied, schema at version 12")); String info = goal(database, "info"); - assertTrue(info, info.contains("cn1: security: 11 migration(s) in cn1_security_schema_history")); + assertTrue(info, info.contains("cn1: security: 12 migration(s) in cn1_security_schema_history")); assertTrue(goal(database, "validate"), goal(database, "validate").contains( "cn1: security: the schema history matches this build's migrations")); assertTrue(goal(database, "migrate").contains("cn1: security: nothing to apply")); diff --git a/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java index a8f06669e3c..57221f9a937 100644 --- a/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java +++ b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java @@ -29,7 +29,6 @@ import java.lang.reflect.Method; import java.net.URI; import java.util.ArrayList; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -74,10 +73,11 @@ synchronized void close() { // close() exists from JDK 21; before it the client's // connections idle out on their own. try { - Method m = clients[iter].getClass().getMethod("shutdownNow"); - m.setAccessible(true); + // Looked up on the public type: the instance's own class + // is internal to java.net.http. + Method m = type("HttpClient").getMethod("shutdownNow"); m.invoke(clients[iter]); - } catch (ReflectiveOperationException | RuntimeException ignored) { + } catch (ReflectiveOperationException | IOException | RuntimeException ignored) { // Not on this JDK. } clients[iter] = null; @@ -107,12 +107,12 @@ private static Object call(Object target, Class owner, String name, Class[] para } if (cause instanceof InterruptedException) { Thread.currentThread().interrupt(); - throw new IOException("The HTTP/2 request was interrupted", cause); + throw new IOException("The HTTP/2 request was interrupted", err); } if (cause instanceof RuntimeException) { throw (RuntimeException) cause; } - throw new IOException(String.valueOf(cause), cause); + throw new IOException(String.valueOf(cause), err); } catch (ReflectiveOperationException err) { throw new IOException("java.net.http has no " + owner.getName() + "." + name, err); } @@ -245,8 +245,8 @@ static Web.Result perform(WebClient client, Clients clients, String method, Stri Object httpHeaders = call(response, responseType, "headers", new Class[0]); Map all = (Map) call(httpHeaders, type("HttpHeaders"), "map", new Class[0]); Map values = new LinkedHashMap(); - for (Iterator it = all.entrySet().iterator(); it.hasNext();) { - Map.Entry e = (Map.Entry) it.next(); + for (Object entry : all.entrySet()) { + Map.Entry e = (Map.Entry) entry; String name = String.valueOf(e.getKey()); if (name.startsWith(":")) { continue; // a pseudo-header, not a field diff --git a/vm/backend/src/com/codename1/backend/Backend.java b/vm/backend/src/com/codename1/backend/Backend.java index d6e07ed79e4..77316dced09 100644 --- a/vm/backend/src/com/codename1/backend/Backend.java +++ b/vm/backend/src/com/codename1/backend/Backend.java @@ -920,9 +920,9 @@ public void decorateFallback(HttpServer.Request request, HttpServer.Response res /// for the reject, the preflight and the response headers alike, so the /// three never disagree about which policy a request is under. Cors corsFor(HttpServer.Request request) { - for (int i = 0 ; i < chain.length ; i++) { - if (chain[i] instanceof com.codename1.backend.web.CorsRoutes) { - Cors own = ((com.codename1.backend.web.CorsRoutes) chain[i]).corsPolicy(request); + for (Object route : chain) { + if (route instanceof com.codename1.backend.web.CorsRoutes) { + Cors own = ((com.codename1.backend.web.CorsRoutes) route).corsPolicy(request); if (own != null) { return own; } diff --git a/vm/backend/src/com/codename1/backend/Cors.java b/vm/backend/src/com/codename1/backend/Cors.java index 5f6777ceae4..3d9a294d051 100644 --- a/vm/backend/src/com/codename1/backend/Cors.java +++ b/vm/backend/src/com/codename1/backend/Cors.java @@ -140,8 +140,8 @@ static Cors fromConfig(Config config) throws IOException { public static Cors policy(String[] origins, String[] methods, String[] headers, String[] exposed, boolean credentials, int maxAge) { List listed = new ArrayList(); - for (int i = 0; i < origins.length; i++) { - String origin = origins[i].trim(); + for (String named : origins) { + String origin = named.trim(); if ("*".equals(origin)) { listed.add(origin); } else if (isOrigin(origin)) { diff --git a/vm/backend/src/com/codename1/backend/Json.java b/vm/backend/src/com/codename1/backend/Json.java index 05c4c972cc8..a8aa37ea0a0 100644 --- a/vm/backend/src/com/codename1/backend/Json.java +++ b/vm/backend/src/com/codename1/backend/Json.java @@ -107,8 +107,8 @@ public static List readList(String json, Class element) throws IOExcep } List in = (List) tree; List out = new ArrayList(in.size()); - for (int iter = 0 ; iter < in.size() ; iter++) { - out.add(convert(in.get(iter), element)); + for (Object item : in) { + out.add(convert(item, element)); } return out; } @@ -551,7 +551,7 @@ private static void writeValue(StringBuilder out, Object value) { try { out.append(new String(sink.bytes(), 0, sink.length(), "UTF-8")); } catch (java.io.UnsupportedEncodingException never) { - throw new IllegalStateException(never.getMessage()); + throw new IllegalStateException(never.toString(), never); } return; } diff --git a/vm/backend/src/com/codename1/backend/log/Level.java b/vm/backend/src/com/codename1/backend/log/Level.java index 419968338d1..3518a144be4 100644 --- a/vm/backend/src/com/codename1/backend/log/Level.java +++ b/vm/backend/src/com/codename1/backend/log/Level.java @@ -35,10 +35,9 @@ public static Level parse(String value) { return null; } String v = value.trim(); - Level[] all = values(); - for (int i = 0; i < all.length; i++) { - if (all[i].name().equalsIgnoreCase(v)) { - return all[i]; + for (Level level : values()) { + if (level.name().equalsIgnoreCase(v)) { + return level; } } // Spring Boot also accepts these two names for the extremes. diff --git a/vm/backend/src/com/codename1/backend/log/MDC.java b/vm/backend/src/com/codename1/backend/log/MDC.java index 1df0a43578d..c93e82d2468 100644 --- a/vm/backend/src/com/codename1/backend/log/MDC.java +++ b/vm/backend/src/com/codename1/backend/log/MDC.java @@ -22,7 +22,6 @@ */ package com.codename1.backend.log; -import java.util.Iterator; import java.util.Map; import java.util.TreeMap; @@ -96,8 +95,7 @@ static void appendTo(StringBuilder out) { } out.append(" {"); boolean first = true; - for (Iterator> it = map.entrySet().iterator(); it.hasNext();) { - Map.Entry e = it.next(); + for (Map.Entry e : map.entrySet()) { if (!first) { out.append(", "); } From 1418c70e4dcb7b2a7a20197270804c6b86c9e69b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:47:23 +0100 Subject: [PATCH 10/25] Backend: refuse a typed long past the range and a wildcard Content-Type Co-Authored-By: Claude Opus 5.5 --- .../java/com/codename1/backend/RoutePolicyTest.java | 3 +++ .../java/com/codename1/impl/backend/JsonCodecTest.java | 10 ++++++++++ vm/backend/src/com/codename1/backend/Json.java | 6 ++++++ .../src/com/codename1/backend/web/MediaTypes.java | 4 +++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java b/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java index 9c502185fb4..6a91b9d8780 100644 --- a/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/RoutePolicyTest.java @@ -80,6 +80,9 @@ void mediaTypesMatchWithWildcardsAndParameters() { assertTrue(MediaTypes.consumable("application/json; charset=UTF-8", "application/json")); assertTrue(MediaTypes.consumable("text/csv", "application/json,text/*")); assertFalse(MediaTypes.consumable(null, "application/json")); + assertFalse(MediaTypes.consumable("*/*", "application/json")); + assertFalse(MediaTypes.consumable("application/*", "application/json")); + assertFalse(MediaTypes.consumable("*", "application/json")); assertFalse(MediaTypes.consumable("application/jsonx", "application/json")); assertTrue(MediaTypes.acceptable(null, "application/json")); assertTrue(MediaTypes.acceptable("*/*", "application/json")); diff --git a/maven/backend/src/test/java/com/codename1/impl/backend/JsonCodecTest.java b/maven/backend/src/test/java/com/codename1/impl/backend/JsonCodecTest.java index 829f760814a..dee98eed60f 100644 --- a/maven/backend/src/test/java/com/codename1/impl/backend/JsonCodecTest.java +++ b/maven/backend/src/test/java/com/codename1/impl/backend/JsonCodecTest.java @@ -96,4 +96,14 @@ void paths() { assertEquals("$[0]", JsonCodec.enter(ROOT, null, 0).toString()); assertEquals("$", JsonCodec.enter(ROOT, null, -1).toString()); } + + @Test + @DisplayName("a typed read refuses a double past the long range instead of clamping it") + void typedLongOutOfRange() throws java.io.IOException { + assertThrows(IllegalArgumentException.class, + () -> com.codename1.backend.Json.read("1e30", Long.class)); + assertThrows(IllegalArgumentException.class, + () -> com.codename1.backend.Json.read("-1e300", Long.class)); + assertEquals(Long.valueOf(5), com.codename1.backend.Json.read("5", Long.class)); + } } diff --git a/vm/backend/src/com/codename1/backend/Json.java b/vm/backend/src/com/codename1/backend/Json.java index a8aa37ea0a0..328ecd5839a 100644 --- a/vm/backend/src/com/codename1/backend/Json.java +++ b/vm/backend/src/com/codename1/backend/Json.java @@ -176,6 +176,12 @@ private static Object builtIn(Object tree, Class type) { if (tree instanceof Double && n.doubleValue() != Math.floor(n.doubleValue())) { throw new IllegalArgumentException("$: expected a whole number, got " + tree); } + if (tree instanceof Double && (n.doubleValue() >= 9.223372036854775807E18 + || n.doubleValue() < -9.223372036854775808E18)) { + // longValue() would clamp 1e30 to Long.MAX_VALUE without a word + throw new IllegalArgumentException("$: " + tree + " is out of range for " + + type.getName()); + } long l = n.longValue(); if (type == Long.class) { return Long.valueOf(l); diff --git a/vm/backend/src/com/codename1/backend/web/MediaTypes.java b/vm/backend/src/com/codename1/backend/web/MediaTypes.java index c166d8f22d3..9f9f21f91a4 100644 --- a/vm/backend/src/com/codename1/backend/web/MediaTypes.java +++ b/vm/backend/src/com/codename1/backend/web/MediaTypes.java @@ -42,7 +42,9 @@ public static boolean consumable(String contentType, String accepted) { return false; } String type = bare(contentType, 0, contentType.length()); - if (type.length() == 0) { + // A request names one concrete type; a wildcard there would satisfy any + // consumes list and let a client past the route's guard. Spring rejects it too. + if (type.length() == 0 || type.indexOf('*') >= 0 || type.indexOf('/') < 0) { return false; } int pos = 0; From aa496a6c85df784cf9b361585f616262a91a5d61 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:57:36 +0100 Subject: [PATCH 11/25] Backend: red-team fixes - per-table index names, body read deadline, MDC reset, plist dates, registry codecs Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/impl/orm/SessionImpl.java | 3 ++ .../developer-guide/Backend-Security.asciidoc | 7 ++- .../security/SecuritySchemaHistoryTest.java | 4 +- .../backend/xml/PropertyListsTest.java | 13 +++++ .../maven/processors/BackendJsonCodecs.java | 5 +- .../com/codename1/backend/JdkHttp2.java | 46 +++++++++++++++- .../src/com/codename1/backend/HttpServer.java | 6 +++ .../com/codename1/backend/LambdaRuntime.java | 1 + .../src/com/codename1/backend/log/MDC.java | 4 +- .../backend/security/SecuritySchema.java | 52 +++++++++---------- .../codename1/backend/xml/PropertyLists.java | 9 +++- 11 files changed, 116 insertions(+), 34 deletions(-) diff --git a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java index ca72c365f3f..c1ce8e77abe 100644 --- a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java +++ b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java @@ -920,6 +920,9 @@ public boolean increment(Class type, Object id, String field, long amount int index = model.queryIndex(field); int version = model.versionIndex(); Attribute a = model.attributes()[index]; + // updatable = false is not checked: as in JPA, it only keeps the column out of + // the UPDATE dirty checking writes, and an increment is an explicit statement, + // the way a JPQL bulk UPDATE of such a column is. if (a.id || a.version || !model.counter(index)) { throw new IllegalArgumentException("Counter must be a non-key integral field: " + field); } diff --git a/docs/developer-guide/Backend-Security.asciidoc b/docs/developer-guide/Backend-Security.asciidoc index 786c2c09975..3047e8a3e9a 100644 --- a/docs/developer-guide/Backend-Security.asciidoc +++ b/docs/developer-guide/Backend-Security.asciidoc @@ -1634,8 +1634,11 @@ cn1.security.table-prefix=auth_ cn1.security.table.cn1_users=app_users ---- -A name must be letters, digits and `_`, optionally after one schema name and a -dot (`auth.users`). The server refuses to start with anything else. An adopted +A name must be letters, digits and `_`, at most 48 characters, and must not +start with a digit. The server refuses to start with anything else. A name is +never qualified by a schema; to keep the tables in another schema, set the +connection's search path. Indexes are named after their tables, so two +applications with different names can share one database. An adopted table must have the columns this chapter's stores use. Every store and every schema version uses the configured names. diff --git a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java index 73ffb4a563d..f8c65c3d319 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java @@ -219,8 +219,8 @@ void configuredTableNamesAreUsedEverywhere() throws Exception { () -> SecuritySchema.setTableName("cn1_users", "1users")); assertThrows(IllegalArgumentException.class, () -> SecuritySchema.setTableName("cn1_users", "a.b.c")); - SecuritySchema.setTableName("cn1_users", "auth.users"); - assertEquals("auth.users", SecuritySchema.table("cn1_users")); + assertThrows(IllegalArgumentException.class, + () -> SecuritySchema.setTableName("cn1_users", "auth.users")); java.util.Properties bad = new java.util.Properties(); bad.setProperty(SecuritySchema.TABLE_PREFIX + "cn1_users", "x\"y"); assertThrows(IllegalArgumentException.class, diff --git a/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java index b62281d9bb9..bb8316725d4 100644 --- a/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java @@ -134,4 +134,17 @@ void pullParserEvents() throws Exception { assertEquals("v", root.childText("i").isEmpty() ? root.children("i").get(1).getText() : "?"); assertEquals("1", root.child("i").getAttribute("n")); } + + @Test + void impossibleDatesAreRefusedAndYearZeroFloors() throws Exception { + assertThrows(XmlException.class, () -> PropertyLists.parse( + "2023-02-29T00:00:00Z")); + assertThrows(XmlException.class, () -> PropertyLists.parse( + "2024-04-31T00:00:00Z")); + assertEquals(951782400000L, ((Date) PropertyLists.parse( + "2000-02-29T00:00:00Z")).getTime()); + // -62167219200000 is 0000-01-01T00:00:00Z + assertEquals(-62167219200000L, ((Date) PropertyLists.parse( + "0000-01-01T00:00:00Z")).getTime()); + } } diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java index c8bba06b825..019637db89f 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendJsonCodecs.java @@ -1691,6 +1691,9 @@ private String cast(String type, String var) { Map sources() { Map out = new LinkedHashMap(); Map owners = new LinkedHashMap(); + // First: reading a class in the registry can ask for the codecs of the + // classes it holds, and the loop below must see those to write them. + String registry = registryWanted ? registrySource() : null; for (Dto dto : dtos.values()) { if (dto.problem != null || (!dto.write && !dto.read)) { continue; @@ -1711,7 +1714,7 @@ Map sources() { } if (registryWanted) { valuesUsed = true; - out.put(REGISTRY, registrySource()); + out.put(REGISTRY, registry); } if (valuesUsed) { out.put(VALUES, valuesSource()); diff --git a/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java index 57221f9a937..8b16e0c0391 100644 --- a/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java +++ b/vm/backend/impl/javase/com/codename1/backend/JdkHttp2.java @@ -51,6 +51,16 @@ final class JdkHttp2 { private static final String P = "java.net.http."; + /// Closes a response body whose read outlived the request's timeout. The + /// client's own timeout ends at the response headers, and a blocked read + /// checks no clock, so closing the stream is what ends a body that trickles. + private static final java.util.concurrent.ScheduledExecutorService DEADLINES = + java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "cn1-http2-deadline"); + t.setDaemon(true); + return t; + }); + private JdkHttp2() { } @@ -221,6 +231,17 @@ static Web.Result perform(WebClient client, Clients clients, String method, Stri Object response = call(http, type("HttpClient"), "send", new Class[] {requestType, handlerType}, request, handler); InputStream in = (InputStream) call(response, responseType, "body", new Class[0]); + java.util.concurrent.atomic.AtomicBoolean expired = + new java.util.concurrent.atomic.AtomicBoolean(); + java.util.concurrent.ScheduledFuture deadline = ceiling <= 0 ? null + : DEADLINES.schedule(() -> { + expired.set(true); + try { + in.close(); + } catch (IOException ignored) { + // the reader sees the closed stream either way + } + }, ceiling, java.util.concurrent.TimeUnit.MILLISECONDS); try { Object version = call(response, responseType, "version", new Class[0]); if (!"HTTP_2".equals(((Enum) version).name())) { @@ -233,7 +254,7 @@ static Web.Result perform(WebClient client, Clients clients, String method, Stri byte[] chunk = new byte[8192]; long total = 0; int n; - while ((n = in.read(chunk)) > 0) { + while ((n = readBody(in, chunk, expired, url, ceiling)) > 0) { total += n; if (total > maxBytes) { throw new IOException("Request to " + Urls.forMessage(url) + " failed: the " @@ -255,10 +276,33 @@ static Web.Result perform(WebClient client, Clients clients, String method, Stri } return new Web.Result(status, buffer.toByteArray(), null, values, "HTTP/2"); } finally { + if (deadline != null) { + deadline.cancel(false); + } in.close(); } } + private static int readBody(InputStream in, byte[] chunk, java.util.concurrent.atomic.AtomicBoolean expired, + String url, int ceiling) throws IOException { + try { + int n = in.read(chunk); + if (expired.get()) { + throw new IOException("closed"); + } + return n; + } catch (IOException err) { + if (expired.get()) { + java.net.SocketTimeoutException timeout = new java.net.SocketTimeoutException( + "Request to " + Urls.forMessage(url) + " timed out after " + ceiling + + "ms reading the response body"); + timeout.initCause(err); + throw timeout; + } + throw err; + } + } + /// See Web.asciiLower; HTTP/2 names are lower case on the wire already, and /// this keeps a JDK that hands them back otherwise from changing that. private static String asciiLower(String value) { diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java index 8354a9d831f..4528d8c57b2 100644 --- a/vm/backend/src/com/codename1/backend/HttpServer.java +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -6451,6 +6451,9 @@ private void serveOneRelease(int fd) { Throwable handlerError = null; try { try { + // A handler that threw past its MDC.remove left its context on + // this thread; the next request must not log under it. + com.codename1.backend.log.MDC.clear(); response = handler.handle(request); if (response == null) { response = fallback(request, 404, "not found"); @@ -6930,6 +6933,9 @@ private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) // owns ending it. boolean spanRegistered = false; try { + // A handler that threw past its MDC.remove left its context on + // this thread; the next request must not log under it. + com.codename1.backend.log.MDC.clear(); response = handler.handle(request); if (response == null) { response = fallback(request, 404, "not found"); diff --git a/vm/backend/src/com/codename1/backend/LambdaRuntime.java b/vm/backend/src/com/codename1/backend/LambdaRuntime.java index 17692a9c3b9..68906fa03bb 100644 --- a/vm/backend/src/com/codename1/backend/LambdaRuntime.java +++ b/vm/backend/src/com/codename1/backend/LambdaRuntime.java @@ -105,6 +105,7 @@ private static boolean answer(Handler handler, String host, int port, Http.Respo String requestId, Span span) { String result; try { + com.codename1.backend.log.MDC.clear(); result = handler.handle(next.getBodyAsString(), requestId); } catch (Exception err) { // The span stays open through the error report too, as it does through diff --git a/vm/backend/src/com/codename1/backend/log/MDC.java b/vm/backend/src/com/codename1/backend/log/MDC.java index c93e82d2468..d3098441427 100644 --- a/vm/backend/src/com/codename1/backend/log/MDC.java +++ b/vm/backend/src/com/codename1/backend/log/MDC.java @@ -84,7 +84,9 @@ public static void remove(String key) { /// Empties this thread's context. public static void clear() { - CONTEXT.set(null); + if (CONTEXT.get() != null) { + CONTEXT.set(null); + } } /// Appends this thread's context as ` {key=value, ...}`, or nothing when empty. diff --git a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java index ca824e2b0b3..2e7a29809f9 100644 --- a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java +++ b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java @@ -212,6 +212,12 @@ public static String table(String defaultName) { return name; } + /// An index's name, derived from its table's configured name so that two + /// schemas configured differently in one database do not collide. + private static String index(String defaultTable, String suffix) { + return table(defaultTable) + "_" + suffix; + } + private static Map names() { Map names = NAMES.get(); if (names == null) { @@ -235,26 +241,21 @@ private static int indexOf(String table) { return -1; } - /// A name that can be written into SQL as it is: one or two dot-separated - /// parts, each a letter or `_` followed by letters, digits and `_`. + /// A name that can be written into SQL as it is: a letter or `_` followed by + /// letters, digits and `_`, at most 48 characters. It is never qualified by a + /// schema: SQLite wants the schema on an index's name and PostgreSQL refuses it + /// there, so a dotted name cannot be indexed portably -- point the connection's + /// search path at the schema instead. The cap leaves room for the index suffix + /// under MySQL's 64 and PostgreSQL's 63 character identifiers. The name is not + /// quoted, so a reserved word such as `user` is the database's to refuse. static String checkIdentifier(String name) { - boolean ok = name != null && name.length() > 0 && name.length() <= 128; - boolean start = true; - int dots = 0; + boolean ok = name != null && name.length() > 0 && name.length() <= 48; for (int iter = 0 ; ok && iter < name.length() ; iter++) { char c = name.charAt(iter); boolean letter = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; - if (c == '.') { - ok = !start && ++dots == 1; - start = true; - } else if (start) { - ok = letter; - start = false; - } else { - ok = letter || (c >= '0' && c <= '9'); - } + ok = letter || iter > 0 && c >= '0' && c <= '9'; } - if (!ok || start) { + if (!ok) { throw new IllegalArgumentException("Not a safe SQL table name: " + name); } return name; @@ -390,8 +391,8 @@ private static String[] apiKeys(Dialect d) { + "id " + key(d) + " NOT NULL, owner " + key(d) + " NOT NULL, scopes " + text(d) + ", prefix " + text(d) + ", last_four " + text(d) + ", revoked " + flag(d) + ", created_at " + moment(d) + ")", - "CREATE UNIQUE INDEX cn1_api_key_id ON " + table("cn1_api_key") + " (id)", - "CREATE INDEX cn1_api_key_owner ON " + table("cn1_api_key") + " (owner)"}; + "CREATE UNIQUE INDEX " + index("cn1_api_key", "id") + " ON " + table("cn1_api_key") + " (id)", + "CREATE INDEX " + index("cn1_api_key", "owner") + " ON " + table("cn1_api_key") + " (owner)"}; } private static String[] persistentLogins(Dialect d) { @@ -399,7 +400,7 @@ private static String[] persistentLogins(Dialect d) { "CREATE TABLE " + table("cn1_persistent_logins") + " (series " + key(d) + " NOT NULL PRIMARY KEY, " + "username_key " + key(d) + " NOT NULL, username " + text(d) + ", token_hash " + text(d) + ", last_used " + moment(d) + ")", - "CREATE INDEX cn1_persistent_logins_user ON " + table("cn1_persistent_logins") + " (username_key)"}; + "CREATE INDEX " + index("cn1_persistent_logins", "user") + " ON " + table("cn1_persistent_logins") + " (username_key)"}; } private static String[] secondFactors(Dialect d) { @@ -441,7 +442,7 @@ private static String[] federatedIdentities(Dialect d) { "CREATE TABLE " + table("cn1_federated_identity") + " (provider " + key(d) + " NOT NULL, subject " + key(d) + " NOT NULL, username_key " + key(d) + " NOT NULL, username " + text(d) + ", created_at " + moment(d) + ", PRIMARY KEY (provider, subject))", - "CREATE INDEX cn1_federated_identity_user ON " + table("cn1_federated_identity") + " (username_key)"}; + "CREATE INDEX " + index("cn1_federated_identity", "user") + " ON " + table("cn1_federated_identity") + " (username_key)"}; } private static String[] registeredClients(Dialect d) { @@ -451,8 +452,7 @@ private static String[] registeredClients(Dialect d) { + ", client_name " + text(d) + ", authentication_methods " + text(d) + ", grant_types " + text(d) + ", redirect_uris " + text(d) + ", scopes " + text(d) + ", settings " + text(d) + ", created_at " + moment(d) + ")", - "CREATE UNIQUE INDEX cn1_oauth2_registered_client_cid ON " - + table("cn1_oauth2_registered_client") + " (client_id)"}; + "CREATE UNIQUE INDEX " + index("cn1_oauth2_registered_client", "cid") + " ON " + table("cn1_oauth2_registered_client") + " (client_id)"}; } /// A grant, and beside it the secrets issued under it -- the authorization @@ -465,12 +465,12 @@ private static String[] authorizations(Dialect d) { + ", grant_type " + text(d) + ", scopes " + text(d) + ", status " + key(d) + " NOT NULL, attributes " + text(d) + ", created_at " + moment(d) + ", expires_at " + moment(d) + ")", - "CREATE INDEX cn1_oauth2_authorization_exp ON " + table("cn1_oauth2_authorization") + " (expires_at)", + "CREATE INDEX " + index("cn1_oauth2_authorization", "exp") + " ON " + table("cn1_oauth2_authorization") + " (expires_at)", "CREATE TABLE " + table("cn1_oauth2_token") + " (token_hash " + key(d) + " NOT NULL PRIMARY KEY, " + "authorization_id " + key(d) + " NOT NULL, kind " + key(d) + " NOT NULL, used " + flag(d) + ", expires_at " + moment(d) + ", polled_at " + moment(d) + ")", - "CREATE INDEX cn1_oauth2_token_auth ON " + table("cn1_oauth2_token") + " (authorization_id)", - "CREATE INDEX cn1_oauth2_token_exp ON " + table("cn1_oauth2_token") + " (expires_at)"}; + "CREATE INDEX " + index("cn1_oauth2_token", "auth") + " ON " + table("cn1_oauth2_token") + " (authorization_id)", + "CREATE INDEX " + index("cn1_oauth2_token", "exp") + " ON " + table("cn1_oauth2_token") + " (expires_at)"}; } /// A user's passkey handle, and their credentials. Byte strings -- the @@ -483,7 +483,7 @@ private static String[] passkeys(Dialect d) { "CREATE TABLE " + table("cn1_webauthn_user") + " (username_key " + key(d) + " NOT NULL PRIMARY KEY, " + "username " + text(d) + ", user_id " + key(d) + " NOT NULL, display_name " + text(d) + ", created_at " + moment(d) + ")", - "CREATE UNIQUE INDEX cn1_webauthn_user_id ON " + table("cn1_webauthn_user") + " (user_id)", + "CREATE UNIQUE INDEX " + index("cn1_webauthn_user", "id") + " ON " + table("cn1_webauthn_user") + " (user_id)", "CREATE TABLE " + table("cn1_webauthn_credential") + " (credential_key " + key(d) + " NOT NULL PRIMARY KEY, credential_id " + text(d) + ", user_id " + key(d) + " NOT NULL, algorithm " + moment(d) + ", public_key " + text(d) @@ -491,6 +491,6 @@ private static String[] passkeys(Dialect d) { + ", backup_eligible " + flag(d) + ", backup_state " + flag(d) + ", transports " + text(d) + ", label " + text(d) + ", created_at " + moment(d) + ", last_used " + moment(d) + ")", - "CREATE INDEX cn1_webauthn_credential_user ON " + table("cn1_webauthn_credential") + " (user_id)"}; + "CREATE INDEX " + index("cn1_webauthn_credential", "user") + " ON " + table("cn1_webauthn_credential") + " (user_id)"}; } } diff --git a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java index 39bdf940a96..9f7b21b9564 100644 --- a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java +++ b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java @@ -228,9 +228,16 @@ private static Date parseDate(String s) throws XmlException { || hour < 0 || hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 60) { throw new XmlException("bad " + s); } + boolean leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + int length = month == 2 ? (leap ? 29 : 28) + : month == 4 || month == 6 || month == 9 || month == 11 ? 30 : 31; + if (day > length) { + throw new XmlException("bad " + s); + } // Days from the civil calendar (H. Hinnant's algorithm), proleptic Gregorian. + // y is -1 for January and February of year 0, so the era floors. int y = month <= 2 ? year - 1 : year; - int era = y / 400; + int era = (y >= 0 ? y : y - 399) / 400; int yoe = y - era * 400; int mp = (month + 9) % 12; int doy = (153 * mp + 2) / 5 + day - 1; From 3007605f967941f903721d7582face0271eef757 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:58:33 +0100 Subject: [PATCH 12/25] JavaAPI: refuse a dotted IPv4 part before :: and non-ASCII hex digits Co-Authored-By: Claude Opus 5.5 --- vm/JavaAPI/src/java/net/InetAddress.java | 7 +++++-- .../com/codename1/tools/translator/bulk/JdkGapChecks.java | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vm/JavaAPI/src/java/net/InetAddress.java b/vm/JavaAPI/src/java/net/InetAddress.java index d4a59adc7a7..2aa172e975b 100644 --- a/vm/JavaAPI/src/java/net/InetAddress.java +++ b/vm/JavaAPI/src/java/net/InetAddress.java @@ -146,7 +146,7 @@ private static byte[] parseIpv6(String s) { int nh; int nt = 0; if (dbl >= 0) { - nh = groups(s.substring(0, dbl), head); + nh = s.substring(0, dbl).indexOf('.') >= 0 ? -1 : groups(s.substring(0, dbl), head); nt = groups(s.substring(dbl + 2), tail); if (nh < 0 || nt < 0 || nh + nt > 7) { return null; @@ -195,7 +195,10 @@ private static int groups(String s, int[] out) { } int v = 0; for (int i = 0; i < g.length(); i++) { - int d = Character.digit(g.charAt(i), 16); + // ASCII only: Character.digit also takes other scripts' digits + char c = g.charAt(i); + int d = c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 + : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; if (d < 0) { return -1; } diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java b/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java index 41aa3605837..144bb159c8a 100644 --- a/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/bulk/JdkGapChecks.java @@ -260,6 +260,8 @@ public void onComplete() { Bulk.p("namedHost", named.getHostName()); Bulk.pThrown("badLength", UnknownHostException.class, () -> InetAddress.getByAddress(new byte[3])); Bulk.pThrown("badV6", UnknownHostException.class, () -> InetAddress.getByName("1::2::3")); + Bulk.pThrown("badV6DottedHead", UnknownHostException.class, + () -> InetAddress.getByName("[1.2.3.4::]")); }); Bulk.cn1Only("JdkGap.inetAddressDoesNotResolveNames", () -> { Bulk.expectEquals("UnknownHostException", From 1c8e9983b0c2cabe95ab7b21633bf98c87599a67 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:08:57 +0100 Subject: [PATCH 13/25] Docs: include the security table-name snippet from docs/demos Co-Authored-By: Claude Opus 5.5 --- .../snippets/developer-guide/backend-security.properties | 5 +++++ docs/developer-guide/Backend-Security.asciidoc | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/demos/common/src/main/snippets/developer-guide/backend-security.properties b/docs/demos/common/src/main/snippets/developer-guide/backend-security.properties index 306288a754c..90f34afc37d 100644 --- a/docs/demos/common/src/main/snippets/developer-guide/backend-security.properties +++ b/docs/demos/common/src/main/snippets/developer-guide/backend-security.properties @@ -42,3 +42,8 @@ cn1.security.schema.enabled=true // tag::backend-security-properties-mfa[] cn1.security.mfa.encryptionKey=replace-with-the-output-of-openssl-rand-base64-32 // end::backend-security-properties-mfa[] + +// tag::backend-security-properties-tables[] +cn1.security.table-prefix=auth_ +cn1.security.table.cn1_users=app_users +// end::backend-security-properties-tables[] diff --git a/docs/developer-guide/Backend-Security.asciidoc b/docs/developer-guide/Backend-Security.asciidoc index 3047e8a3e9a..19c8d081798 100644 --- a/docs/developer-guide/Backend-Security.asciidoc +++ b/docs/developer-guide/Backend-Security.asciidoc @@ -1630,8 +1630,7 @@ of all of them. The per-table setting wins: [source,properties] ---- -cn1.security.table-prefix=auth_ -cn1.security.table.cn1_users=app_users +include::../demos/common/src/main/snippets/developer-guide/backend-security.properties[tag=backend-security-properties-tables,indent=0] ---- A name must be letters, digits and `_`, at most 48 characters, and must not From 5866191047084d3070726226b0373be4667f26c0 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:55:35 +0100 Subject: [PATCH 14/25] Backend security: read the table names lazily so a server without stores never links SecuritySchema Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/backend/security/Access.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/vm/backend/src/com/codename1/backend/security/Access.java b/vm/backend/src/com/codename1/backend/security/Access.java index d6cbb25d541..f22dbfe5762 100644 --- a/vm/backend/src/com/codename1/backend/security/Access.java +++ b/vm/backend/src/com/codename1/backend/security/Access.java @@ -35,7 +35,6 @@ final class Access extends SecurityAccess { public HttpSecurity httpSecurity(Config config, Object[] beans) { com.codename1.impl.backend.security.SecuritySupport.developmentProfile( config != null && config.isDevelopmentProfile()); - tableNames(config); return new HttpSecurity(config, beans); } @@ -44,7 +43,6 @@ public HttpSecurity httpSecurity(Config config, Object[] beans, String[] names, boolean[] primary) { com.codename1.impl.backend.security.SecuritySupport.developmentProfile( config != null && config.isDevelopmentProfile()); - tableNames(config); return new HttpSecurity(config, beans, names, primary); } @@ -99,18 +97,4 @@ public String testCsrf(boolean valid) { public void clearTestCsrf() { CsrfFilter.testToken(null); } - - // The server's configuration names the security tables; read here so every - // repository uses them without a configuration of its own. - private static void tableNames(Config config) { - if (config == null) { - return; - } - try { - SecuritySchema.configure(config); - } catch (java.io.IOException err) { - throw new IllegalStateException("The security table names could not be read: " - + err.getMessage(), err); - } - } } From 895e75265b954d0d1bbba8429f435a6d04e67dc8 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:57:48 +0100 Subject: [PATCH 15/25] Docs: Vale wording in the security table-name section Co-Authored-By: Claude Opus 5.5 --- docs/developer-guide/Backend-Security.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer-guide/Backend-Security.asciidoc b/docs/developer-guide/Backend-Security.asciidoc index 19c8d081798..72564b7f093 100644 --- a/docs/developer-guide/Backend-Security.asciidoc +++ b/docs/developer-guide/Backend-Security.asciidoc @@ -1626,7 +1626,7 @@ are the same on SQLite, PostgreSQL, and MySQL. To use tables you already have, give their names in `application.properties`. A per-table setting names one table, and a prefix replaces the `cn1_` in front -of all of them. The per-table setting wins: +of every name. The per-table setting wins: [source,properties] ---- From 4024e09616b2a1b609a64cda9ea51538759d946a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:41:45 +0100 Subject: [PATCH 16/25] ORM: @Column(collation), and accept a declared coarser @Temporal column Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/annotations/Column.java | 12 +++++ .../codename1/annotations/db/Temporal.java | 8 +++- .../src/com/codename1/impl/orm/Attribute.java | 11 +++++ .../com/codename1/impl/orm/SessionImpl.java | 48 ++++++++++++++++--- .../processors/OrmAnnotationProcessor.java | 36 ++++++++++---- .../OrmAnnotationProcessorTest.java | 47 +++++++++++++++++- 6 files changed, 144 insertions(+), 18 deletions(-) diff --git a/CodenameOne/src/com/codename1/annotations/Column.java b/CodenameOne/src/com/codename1/annotations/Column.java index 7a1bdc0a130..29619638f0d 100644 --- a/CodenameOne/src/com/codename1/annotations/Column.java +++ b/CodenameOne/src/com/codename1/annotations/Column.java @@ -70,4 +70,16 @@ /// A change made to the field alone is ignored: it neither writes the column /// nor marks the entity dirty. boolean updatable() default true; + /// The collation of the column, appended to its DDL as `COLLATE ` + /// (quoted on PostgreSQL), and the one the schema report then holds an + /// existing column to instead of requiring a case-sensitive key collation. + /// Letters, digits and underscores only, as `utf8mb4_0900_ai_ci` on MySQL, + /// `NOCASE` on SQLite or a collation created on PostgreSQL. + /// + /// A case-insensitive collation on a key makes the database treat `"A"` and + /// `"a"` as one key. The session's identity cache still compares ids with + /// `String.equals`, so look an entity up by the spelling it is stored with: + /// another one finds the row but loads a second instance of it. Hibernate + /// has the same caveat. + String collation() default ""; } diff --git a/CodenameOne/src/com/codename1/annotations/db/Temporal.java b/CodenameOne/src/com/codename1/annotations/db/Temporal.java index a62831b5c7e..6c2c8847aaf 100644 --- a/CodenameOne/src/com/codename1/annotations/db/Temporal.java +++ b/CodenameOne/src/com/codename1/annotations/db/Temporal.java @@ -52,8 +52,12 @@ /// not change what is stored. A `TIMESTAMP WITH TIME ZONE` column is read with /// the offset the server sends. /// - Precision is the **millisecond**, which is all a Date holds. A column with -/// finer precision has the extra digits dropped on read; one with coarser -/// precision rounds or truncates on write, as that engine does. +/// finer precision has the extra digits dropped on read. A coarser column, +/// such as a plain MySQL `DATETIME`, is accepted only when the field declares +/// it, `@Column(type = "DATETIME")`: the declaration is the opt-in to losing +/// the milliseconds. The engine then rounds or truncates on write, as it does +/// for Hibernate, so on MySQL a value may read back one second later. MySQL's +/// `TIMESTAMP` is still refused, since it ends in 2038. /// - On SQLite, which has no native date type, the value is text in the form /// `2026-10-16 09:30:00.000`, which sorts in time order. Rows written in a /// different textual form are read when they are ISO 8601, but compare as text diff --git a/CodenameOne/src/com/codename1/impl/orm/Attribute.java b/CodenameOne/src/com/codename1/impl/orm/Attribute.java index f42db495bf7..2f16efd73ab 100644 --- a/CodenameOne/src/com/codename1/impl/orm/Attribute.java +++ b/CodenameOne/src/com/codename1/impl/orm/Attribute.java @@ -55,6 +55,10 @@ public final class Attribute { public final boolean insertable; /// False for `@Column(updatable = false)`: no dirty-checking UPDATE names it. public final boolean updatable; + /// Null, or the `@Column(collation)` name: the DDL appends it, and the + /// schema report holds the column to it instead of to a case-sensitive key + /// collation. + public final String collation; /// The kind that decides SQL semantics -- collation, LIKE, the text /// functions -- as opposed to [#kind], which decides how a value is bound and /// read. A `@Temporal` column exchanges text through its converter but is a @@ -76,6 +80,13 @@ public Attribute(String field, String column, int kind, boolean id, boolean gene } public Attribute(String field, String column, int kind, boolean id, boolean generated, boolean nullable, boolean version, String declaredType, int temporal, boolean insertable, boolean updatable) { + this(field, column, kind, id, generated, nullable, version, declaredType, temporal, insertable, updatable, + null); + } + public Attribute(String field, String column, int kind, boolean id, boolean generated, boolean nullable, + boolean version, String declaredType, int temporal, boolean insertable, boolean updatable, + String collation) { + this.collation = collation == null || collation.length() == 0 ? null : collation; this.insertable = insertable; this.updatable = updatable; this.temporal = temporal; diff --git a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java index c1ce8e77abe..2f96e1548b4 100644 --- a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java +++ b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java @@ -1010,7 +1010,8 @@ public void createTables() { throw new PersistenceException("A @Temporal(TIMESTAMP) field needs DATETIME(3) on MySQL, " + "not " + a.declaredType + ": " + model.table() + "." + a.column); } - statement.append(columnType(model, i)).append(a.nullable && !a.id ? "" : " NOT NULL"); + statement.append(columnType(model, i)).append(collateClause(a)) + .append(a.nullable && !a.id ? "" : " NOT NULL"); } } if (!attrs[model.idIndex()].generated) { @@ -1169,6 +1170,24 @@ private String columnType(EntityModel model, int index) { int end = declaration.indexOf(" PRIMARY KEY"); return end < 0 ? sql.columnType(attr.kind) : declaration.substring(0, end).replace(" NOT NULL", ""); } + /// ` COLLATE ` for a `@Column(collation)` field, else empty. + /// PostgreSQL names a collation as an identifier, which folds unquoted. + private String collateClause(Attribute attr) { + if (attr.collation == null) { + return ""; + } + return " COLLATE " + ("postgresql".equals(sql.dialect()) ? q(attr.collation) : attr.collation); + } + /// Whether a found collation is the declared one. The PostgreSQL describe + /// prefixes a nondeterministic one, which is part of what was declared. + /// SQLite reports a collation for key columns only; an unknown one passes. + static boolean declaredCollation(String declared, String found) { + if (found == null) { + return true; + } + String name = found.startsWith("nondeterministic ") ? found.substring(17) : found; + return declared.equalsIgnoreCase(name) || declared.equalsIgnoreCase(found); + } /// The native column of a @Temporal field. DATETIME rather than TIMESTAMP on /// MySQL: its TIMESTAMP ends in 2038 and, unless explicit_defaults_for_timestamp /// is on, gives the first such column an ON UPDATE CURRENT_TIMESTAMP that @@ -1221,9 +1240,12 @@ private boolean storageCompatible(EntityModel model, int index, String found, St return false; } if (attr.temporal == Attribute.TEMPORAL_TIMESTAMP && family == 5 - && temporalKind(found) == Attribute.TEMPORAL_TIMESTAMP && fractionDigits(found) < 3) { + && temporalKind(found) == Attribute.TEMPORAL_TIMESTAMP && fractionDigits(found) < 3 + && (attr.declaredType == null || fractionDigits(attr.declaredType) > fractionDigits(found))) { // The converter writes milliseconds; DATETIME or TIMESTAMP(0) // rounds them away, so a value does not read back as written. + // Declaring the coarser type, as `@Column(type = "DATETIME")`, + // accepts that: the converter truncates to it (see Values). return false; } if (family == 4 && binaryCapacity(found) < binaryCapacity(expected)) { @@ -1397,7 +1419,13 @@ public SchemaReport schemaReport() { // read back, so adopting such a schema keeps its semantics. boolean distinct = attr.id || uniqueIndexed(model, attr); String role = attr.id ? "Identifier " : "Unique column "; - if (distinct && attr.kind == Attribute.TEXT && found.length > 5 + if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5])) { + errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, model.table(), attr.column, + name + " uses collation " + found[5] + ", not the declared " + attr.collation)); + } + // A declared collation is the application's choice of key + // equality, as in Hibernate. + if (distinct && attr.kind == Attribute.TEXT && found.length > 5 && attr.collation == null && !distinctKeyCollation((String) found[5])) { errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, model.table(), attr.column, role + name + " uses collation " + found[5] @@ -1486,7 +1514,8 @@ private void collectionKeys(List expected, String[] names, EntityMode int[] ids = model.idIndexes(); for (int i = 0; i < names.length; i++) { Attribute id = model.attributes()[ids[i]]; - expected.add(new Attribute("", names[i], id.kind, primary, false, false, false, id.declaredType)); + expected.add(new Attribute("", names[i], id.kind, primary, false, false, false, id.declaredType, 0, + true, true, id.collation)); } } /// `element` is the value column of an element collection, null @@ -1512,7 +1541,13 @@ private void validateCollectionColumns(String table, List expected, L // SQLite lets a composite primary key column hold NULL, so only its // NOT NULL flag says the column is required there. boolean required = ((Number) found[2]).intValue() != 0 || primary && !"sqlite".equals(sql.dialect()); - if (attr.id && attr.kind == Attribute.TEXT && found.length > 5 && !distinctKeyCollation((String) found[5])) { + if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5])) { + errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, table, attr.column, + "Collection column " + name + " uses collation " + found[5] + ", not the declared " + + attr.collation)); + } + if (attr.id && attr.kind == Attribute.TEXT && found.length > 5 && attr.collation == null + && !distinctKeyCollation((String) found[5])) { errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, table, attr.column, "Collection key " + name + " uses collation " + found[5] + ", which folds case or pads; keys need a case-sensitive NO PAD collation")); @@ -3343,7 +3378,8 @@ private void appendJoinDefinition(StringBuilder statement, String[] names, Entit int end = keyType.indexOf(" PRIMARY KEY"); type = end < 0 ? sql.columnType(attr.kind) : keyType.substring(0, end).replace(" NOT NULL", ""); } - statement.append(q(names[i])).append(' ').append(type).append(" NOT NULL"); + // A foreign key compares as the key it names, so it takes its collation. + statement.append(q(names[i])).append(' ').append(type).append(collateClause(attr)).append(" NOT NULL"); } } String keyCondition(EntityModel model, String alias) { diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java index 737f5f8036b..7a38abfc69b 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java @@ -568,6 +568,15 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from nullable = col.getBoolOrDefault("nullable", true); pf.insertable = col.getBoolOrDefault("insertable", true); pf.updatable = col.getBoolOrDefault("updatable", true); + String collation = col.getString("collation"); + if (collation != null && collation.length() > 0) { + if (!collationName(collation)) { + // Spliced into DDL as it is. + ctx.error(cls, "@Column(collation) must be letters, digits and underscores: " + + pf.fieldName + " declares \"" + collation + "\""); + } + pf.collation = collation; + } if ((!pf.insertable || !pf.updatable) && (f.getAnnotation(ID_DESC) != null || pf.version)) { ctx.error(cls, "@Column(insertable/updatable=false) cannot be used on an @Id or @Version field: " + pf.fieldName); } @@ -1172,6 +1181,16 @@ && lowerOrDigit(name.charAt(i-1)) && lowerOrDigit(name.charAt(i+1))) return out.toString().toLowerCase(java.util.Locale.ROOT); } + /// Whether a `@Column(collation)` name is safe to splice into DDL. + static boolean collationName(String name) { + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_')) { + return false; + } + } + return name.length() > 0; + } /// The java.time field types the ORM maps, by descriptor, to the simple name /// JavaTimeConverter's nested converters start with. private static final java.util.Map JAVA_TIME_TYPES=new java.util.HashMap(); @@ -1190,12 +1209,10 @@ static boolean holdsTemporal(String type,int temporal) { if("text".equals(token)) return true; if(temporal==1) return "date".equals(token); if(!"timestamp".equals(token) && !"timestamptz".equals(token) && !"datetime".equals(token)) return false; - // The converter writes milliseconds, so the column has to keep three - // fractional digits: MySQL's DATETIME defaults to none, PostgreSQL's - // TIMESTAMP to six. - java.util.regex.Matcher precision=java.util.regex.Pattern.compile("\\(\\s*(\\d+)\\s*\\)").matcher(type); - int digits=precision.find() ? Integer.parseInt(precision.group(1)) : "datetime".equals(token) ? 0 : 6; - return digits>=3; + // The converter writes milliseconds. A coarser type -- MySQL's DATETIME + // keeps none -- is the developer's explicit choice to lose them, as + // Hibernate allows, and the engine rounds on write. + return true; } private static boolean lowerOrDigit(char c) { return Character.isLowerCase(c) || Character.isDigit(c); } @@ -1546,8 +1563,9 @@ private static String generateModelSource(EntityClass ec, boolean backend) { .append(", ").append(f.isId).append(", ").append(f.isId && f.autoIncrement) .append(", ").append(f.nullable).append(", ").append(f.version).append(", ") .append(f.explicitSqlType==null?"null":"\""+escape(f.explicitSqlType)+"\"") - .append(f.temporal==0 && f.insertable && f.updatable?"":", "+f.temporal) - .append(f.insertable && f.updatable?"":", "+f.insertable+", "+f.updatable).append(")") + .append(f.temporal==0 && f.insertable && f.updatable && f.collation==null?"":", "+f.temporal) + .append(f.insertable && f.updatable && f.collation==null?"":", "+f.insertable+", "+f.updatable) + .append(f.collation==null?"":", \""+f.collation+"\"").append(")") .append(i + 1 < ec.fields.size() ? ",\n" : "\n"); } sb.append(" };\n public Class<").append(ec.binaryName).append("> type() { return ") @@ -3179,6 +3197,8 @@ static final class PersistedField { boolean unique; /// @Column(insertable/updatable); both true unless the developer said otherwise. boolean insertable = true, updatable = true; + /// @Column(collation), or null. + String collation; RelationField relation; int relationPart; PropertyTypeKind kind; diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java index cfe47f889df..5b823b775f4 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java @@ -127,6 +127,49 @@ private void generatedRepository(boolean backend) throws Exception { finally { session.close();if(server!=null)server.close();if(client!=null)client.close();loader.close(); } } + /// `@Column(collation)` reaches the DDL and the schema report: a NOCASE key + /// on SQLite treats "ABC" and "abc" as one key, and validates instead of + /// being refused as a case-folding key. A `@Temporal` field declared as a + /// plain `DATETIME` is accepted too. + @Test + public void declaredCollationAndCoarseTemporal() throws Exception { + declaredCollation(false); + declaredCollation(true); + } + private void declaredCollation(boolean backend) throws Exception { + File classes=tmp.newFolder(); + Map sources=new java.util.LinkedHashMap(); + sources.put("collated.Code","package collated; import com.codename1.annotations.*; import com.codename1.annotations.db.*; @Entity public class Code { @Id(autoIncrement=false) @Column(collation=\"NOCASE\") public String code; @Temporal(TemporalType.TIMESTAMP) @Column(type=\"DATETIME\") public java.util.Date at; }"); + sources.put("collated.Check","package collated; import com.codename1.orm.session.*; public class Check { public static void run(Session s) { " + + "s.createTables(); if(!s.schemaReport().getIssues().isEmpty()) throw new AssertionError(\"report \"+s.schemaReport().getIssues()); s.validateSchema(); " + + "s.beginTransaction(); Code a=new Code(); a.code=\"ABC\"; a.at=new java.util.Date(1000); s.persist(a); s.commitTransaction(); s.clear(); " + + "if(s.find(Code.class,\"ABC\").at.getTime()!=1000) throw new AssertionError(\"temporal\"); " + + "boolean refused=false; try { s.beginTransaction(); Code b=new Code(); b.code=\"abc\"; s.persist(b); s.commitTransaction(); } catch(RuntimeException e) { refused=true; s.rollbackTransaction(); } " + + "if(!refused) throw new AssertionError(\"NOCASE key took a second spelling\"); } }"); + JavaSourceCompiler.compile(sources,classes,Arrays.asList(testClassesDir())); + ProcessorContext ctx=runProcessor(classes,backend ? backendClasspath() : Collections.emptyList());assertFalse(ctx.getErrors().toString(),ctx.hasErrors()); + java.net.URLClassLoader loader=new java.net.URLClassLoader(new URL[]{classes.toURI().toURL()},getClass().getClassLoader()); + loader.loadClass(backend ? "cn1app.BackendDaoBootstrap" : "cn1app.DaoBootstrap").newInstance(); + com.codename1.backend.orm.EntityManager server=backend ? com.codename1.backend.orm.EntityManager.open(com.codename1.backend.Database.open(":memory:")) : null; + com.codename1.orm.EntityManager client=backend ? null : com.codename1.orm.EntityManager.open(new com.codename1.impl.javase.SEDatabase(java.sql.DriverManager.getConnection("jdbc:sqlite::memory:"))); + com.codename1.orm.session.Session session=backend ? server.openSession() : client.openSession(); + try { loader.loadClass("collated.Check").getMethod("run",com.codename1.orm.session.Session.class).invoke(null,session); } + finally { session.close();if(server!=null)server.close();if(client!=null)client.close();loader.close(); } + } + + /// A collation is spliced into DDL, so the processor refuses anything but a name. + @Test + public void collationMustBeAName() throws Exception { + File classes=tmp.newFolder(); + Map sources=new java.util.LinkedHashMap(); + sources.put("badcollation.Code","package badcollation; import com.codename1.annotations.*; @Entity public class Code { @Id(autoIncrement=false) @Column(collation=\"NOCASE; DROP TABLE x\") public String code; }"); + JavaSourceCompiler.compile(sources,classes,Arrays.asList(testClassesDir())); + ProcessorContext ctx=runProcessor(classes,Collections.emptyList()); + assertTrue(ctx.getErrors().toString(),ctx.getErrors().toString().contains("@Column(collation) must be letters")); + assertTrue(OrmAnnotationProcessor.collationName("utf8mb4_0900_ai_ci")); + assertFalse(OrmAnnotationProcessor.collationName("en-x-icu")); + } + /// `@Column(insertable = false, updatable = false)` is a column only the /// application's own statement writes: a compare-and-swap claim sets it, and /// neither the insert nor a later save of the entity overwrites it. `@Lob` is @@ -1436,8 +1479,8 @@ public void floatingPointSetsKeepTheirPositionColumn() throws Exception { public void temporalFieldsRejectAColumnTypeThatCannotHoldThem() throws Exception { String[][] cases={{"TIMESTAMP","BIGINT","false"},{"TIMESTAMP","DATE","false"},{"DATE","TIMESTAMP(3)","false"},{"DATE","varchar(20)","false"}, {"TIMESTAMP","TIMESTAMP(3)","true"},{"TIMESTAMP","datetime(3)","true"},{"TIMESTAMP","TIMESTAMPTZ","true"},{"DATE","date","true"},{"DATE","TEXT","true"}, - // Milliseconds survive only with three fractional digits. - {"TIMESTAMP","TIMESTAMP(0)","false"},{"TIMESTAMP","DATETIME","false"},{"TIMESTAMP","datetime(6)","true"},{"TIMESTAMP","TIMESTAMP","true"}}; + // A declared coarser type opts in to losing the milliseconds. + {"TIMESTAMP","TIMESTAMP(0)","true"},{"TIMESTAMP","DATETIME","true"},{"TIMESTAMP","datetime(6)","true"},{"TIMESTAMP","TIMESTAMP","true"}}; for(String[] c:cases) { File classes=tmp.newFolder();Map sources=new java.util.LinkedHashMap(); sources.put("temporaltype.Entry","package temporaltype; import com.codename1.annotations.*; import com.codename1.annotations.db.*; import java.util.Date; " From 56a69850e9851a0e434f8df73fe4ce2d0908bf2a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:44:39 +0100 Subject: [PATCH 17/25] ORM: ElementCollection joinColumn and type; unwrap Hibernate-quoted names ElementCollection can name its owner column and declare the value column's SQL type, which reaches CREATE TABLE and the schema report. A name wrapped in backticks or double quotes is unwrapped, since the runtime quotes every identifier itself and the quotes otherwise became part of the name. Co-Authored-By: Claude Opus 5.5 --- .../annotations/db/ElementCollection.java | 10 +++++ .../com/codename1/impl/orm/Relationship.java | 9 +++++ .../com/codename1/impl/orm/SessionImpl.java | 9 +++-- .../Annotation-SQLite-ORM.asciidoc | 5 +++ .../processors/OrmAnnotationProcessor.java | 38 ++++++++++++++----- .../OrmAnnotationProcessorTest.java | 26 +++++++++++++ 6 files changed, 84 insertions(+), 13 deletions(-) diff --git a/CodenameOne/src/com/codename1/annotations/db/ElementCollection.java b/CodenameOne/src/com/codename1/annotations/db/ElementCollection.java index c402fd76b56..c57d27726ec 100644 --- a/CodenameOne/src/com/codename1/annotations/db/ElementCollection.java +++ b/CodenameOne/src/com/codename1/annotations/db/ElementCollection.java @@ -33,6 +33,16 @@ /// Names the scalar value column in the collection table. /// @return column name; element_value by default String column() default "element_value"; + /// Names the column that holds the owner's id in the collection table. + /// Empty keeps `JoinTable.joinColumn` when that is set and `owner_id` + /// otherwise; naming the column in both places is a build error. + /// @return owner column name; empty for the default + String joinColumn() default ""; + /// The SQL type of the value column, written into `CREATE TABLE` as it + /// is, like `Column.type` -- for example `LONGTEXT` -- and validated against + /// the existing column the same way. + /// @return SQL type; empty for the dialect's type for the element class + String type() default ""; /// Names the column that stores each element's position, which keys the /// table together with the owner and keeps a List's order and duplicates. /// diff --git a/CodenameOne/src/com/codename1/impl/orm/Relationship.java b/CodenameOne/src/com/codename1/impl/orm/Relationship.java index 0bf6e9dd56b..7b58d7afa97 100644 --- a/CodenameOne/src/com/codename1/impl/orm/Relationship.java +++ b/CodenameOne/src/com/codename1/impl/orm/Relationship.java @@ -42,6 +42,8 @@ public final class Relationship { public final boolean orphanRemoval; public final boolean unique; public final boolean element; + /// The SQL type an element collection declares for its value column, empty for the default. + public final String elementType; public final int column; public final int cascade; public static final int PERSIST = 1; @@ -69,7 +71,14 @@ public Relationship(String field, Class target, boolean many, boolean lazy, int public Relationship(String field, Class target, boolean many, boolean lazy, int column, String mappedBy, String joinTable, String joinColumn, String inverseJoinColumn, int cascade, boolean orphanRemoval, boolean unique, String mapKey, String orderColumn, String orderBy, boolean element) { + this(field, target, many, lazy, column, mappedBy, joinTable, joinColumn, inverseJoinColumn, cascade, + orphanRemoval, unique, mapKey, orderColumn, orderBy, element, ""); + } + public Relationship(String field, Class target, boolean many, boolean lazy, int column, String mappedBy, + String joinTable, String joinColumn, String inverseJoinColumn, int cascade, boolean orphanRemoval, + boolean unique, String mapKey, String orderColumn, String orderBy, boolean element, String elementType) { this.element = element; + this.elementType = elementType == null ? "" : elementType; this.mapKey = mapKey; this.orderColumn = orderColumn; this.orderBy = orderBy; diff --git a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java index 2f96e1548b4..fa9d0d5a669 100644 --- a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java +++ b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java @@ -1489,7 +1489,8 @@ public SchemaReport schemaReport() { if (relation.element) { boolean keyed = keyedByValue(relation); element = new Attribute("", relation.inverseJoinColumn, Elements.kind(relation.target), - keyed, false, !keyed, false); + keyed, false, !keyed, false, + relation.elementType.length() == 0 ? null : relation.elementType); expected.add(element); if (relation.mapKey.length() > 0) { expected.add(new Attribute("", relation.mapKey, Attribute.TEXT, true, false, false, false)); @@ -2537,7 +2538,8 @@ private void createElementTable(EntityModel owner, Relationship relation, List 0 ? relation.elementType + : sql.assignedKeyColumn(Elements.kind(relation.target)) .replace(" PRIMARY KEY", "").replace(" NOT NULL", ""); definition.append(", ").append(q(relation.inverseJoinColumn)).append(' ').append(type) .append(" NOT NULL, PRIMARY KEY (").append(quoted(columns)).append(", ") @@ -2548,7 +2550,8 @@ private void createElementTable(EntityModel owner, Relationship relation, List 0 ? relation.elementType + : sql.columnType(Elements.kind(relation.target))); if (relation.mapKey.length() > 0) { String type = sql.assignedKeyColumn(Attribute.TEXT).replace(" PRIMARY KEY", "").replace(" NOT NULL", ""); definition.append(", ").append(q(relation.mapKey)).append(' ').append(type).append(" NOT NULL"); diff --git a/docs/developer-guide/Annotation-SQLite-ORM.asciidoc b/docs/developer-guide/Annotation-SQLite-ORM.asciidoc index 44657ec1876..6157d8ac840 100644 --- a/docs/developer-guide/Annotation-SQLite-ORM.asciidoc +++ b/docs/developer-guide/Annotation-SQLite-ORM.asciidoc @@ -372,6 +372,11 @@ List, Set, and Map with String keys. Values may be String, boxed primitive types or `java.util.Date`. Lists preserve order and duplicate values. Map columns can be named with `@MapKeyColumn`; `@JoinTable` names the collection table and its owner column, and `@ElementCollection(column=...)` names the value column. +`@ElementCollection(joinColumn=...)` names the owner column without a +`@JoinTable`, and `type=...` sets the value column's SQL type, such as +`LONGTEXT`, the way `@Column(type=...)` does. Names are quoted for you, so a +name such as `signal` needs no backticks, and a name wrapped in backticks or +double quotes has them removed. A List or Set table has an `element_position` column by default. `@ElementCollection(positionColumn="")` maps a Set to a table with no position column, keyed by owner and value, which is the usual shape of an existing tag diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java index 7a38abfc69b..55da1b199b7 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java @@ -408,7 +408,7 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from : ec.packageName + "." + ec.daoSimpleName; Object indexes=entityAnn.get("indexes"); if(indexes instanceof List) for(Object index:(List)indexes) ec.indexes.add((AnnotationValues)index); - String table = entityAnn.getString("table"); + String table = unquoteName(entityAnn.getString("table")); ec.tableName = (table == null || table.length() == 0) ? physical(ec.simpleName) : table; if (backend && tooLongForAnEngine(ec.tableName)) { ctx.error(cls, "@Entity " + ec.binaryName + " is stored in a table named '" @@ -563,7 +563,7 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from String colType = null; boolean nullable = true; if (col != null) { - colName = col.getString("name"); + colName = unquoteName(col.getString("name")); colType = col.getString("type"); nullable = col.getBoolOrDefault("nullable", true); pf.insertable = col.getBoolOrDefault("insertable", true); @@ -1294,7 +1294,7 @@ private RelationField relation(FieldInfo field, EntityClass owner, ProcessorCont Object position=annotation.get("positionColumn"); if(position instanceof String) { if(orderColumn!=null) ctx.error("Choose OrderColumn or ElementCollection.positionColumn: "+relation.field); - relation.orderColumn=(String)position; + relation.orderColumn=unquoteName((String)position); if(relation.orderColumn.length()==0 && !"Ljava/util/Set;".equals(field.getDescriptor())) ctx.error("Only a Set element collection can omit its position column: "+relation.field); } else if(relation.orderColumn.length()==0) relation.orderColumn="element_position"; @@ -1348,14 +1348,14 @@ private RelationField relation(FieldInfo field, EntityClass owner, ProcessorCont ctx.error("A Set of "+relation.target+" needs its position column, since its values cannot key the table: "+relation.field); AnnotationValues join=field.getAnnotation(prefix+"JoinColumn;"); String joinDefault=physical(relation.field)+"_id"; - relation.columnName=join==null?joinDefault:join.getStringOrDefault("name",joinDefault); + relation.columnName=join==null?joinDefault:unquoteName(join.getStringOrDefault("name",joinDefault)); if(relation.columnName.length()==0) relation.columnName=joinDefault; relation.nullable=annotation.getBoolOrDefault("optional",true) && (join==null || join.getBoolOrDefault("nullable",true)); AnnotationValues table=field.getAnnotation(prefix+"JoinTable;"); String tableDefault=owner.tableName+"_"+physical(relation.field); - relation.table=table==null?tableDefault:table.getStringOrDefault("name",tableDefault); - relation.ownerColumn=table==null?"owner_id":table.getStringOrDefault("joinColumn","owner_id"); - relation.targetColumn=table==null?"target_id":table.getStringOrDefault("inverseJoinColumn","target_id"); + relation.table=table==null?tableDefault:unquoteName(table.getStringOrDefault("name",tableDefault)); + relation.ownerColumn=table==null?"owner_id":unquoteName(table.getStringOrDefault("joinColumn","owner_id")); + relation.targetColumn=table==null?"target_id":unquoteName(table.getStringOrDefault("inverseJoinColumn","target_id")); if(relation.table.length()==0) relation.table=tableDefault; if(relation.ownerColumn.length()==0) relation.ownerColumn="owner_id"; if(relation.targetColumn.length()==0) relation.targetColumn="target_id"; @@ -1363,7 +1363,15 @@ private RelationField relation(FieldInfo field, EntityClass owner, ProcessorCont if(relation.many && join!=null) ctx.error("Use mappedBy or JoinTable for a to-many association: "+relation.field); if(!relation.many && table!=null) ctx.error("To-one associations use JoinColumn: "+relation.field); if(relation.element) { - relation.targetColumn=annotation.getStringOrDefault("column","element_value"); + relation.targetColumn=unquoteName(annotation.getStringOrDefault("column","element_value")); + if(relation.targetColumn.length()==0) relation.targetColumn="element_value"; + String owning=unquoteName(annotation.getStringOrDefault("joinColumn","")); + if(owning.length()>0) { + if(table!=null && table.getStringOrDefault("joinColumn","").length()>0) + ctx.error("Choose JoinTable.joinColumn or ElementCollection.joinColumn: "+relation.field); + relation.ownerColumn=owning; + } + relation.elementType=annotation.getStringOrDefault("type",""); if(map && (field.getSignature()==null || !field.getSignature().contains(" sources=new java.util.LinkedHashMap(); + // Hibernate-style quoting: the runtime quotes every identifier itself, so the + // quotes must not become part of the name. + sources.put("ecnames.Feed","package ecnames; import com.codename1.annotations.*; import com.codename1.annotations.db.*; @Entity(table=\"`ec_feed`\") public class Feed { @Id public long id; @Column(name=\"`signal`\") public String signal; @Column(name=\"\\\"order\\\"\") public int order; @ElementCollection(joinColumn=\"`feed_ref`\",column=\"\\\"note\\\"\",type=\"LONGTEXT\") @JoinTable(name=\"`ec_feed_note`\") public java.util.List notes=new java.util.ArrayList(); }"); + JavaSourceCompiler.compile(sources,classes,Arrays.asList(testClassesDir())); + ProcessorContext ctx=runProcessor(classes,backendClasspath());assertFalse(ctx.getErrors().toString(),ctx.hasErrors()); + java.net.URLClassLoader loader=new java.net.URLClassLoader(new URL[]{classes.toURI().toURL()},getClass().getClassLoader());loader.loadClass("cn1app.BackendDaoBootstrap").newInstance();Class type=loader.loadClass("ecnames.Feed"); + com.codename1.backend.orm.EntityManager em=com.codename1.backend.orm.EntityManager.open(com.codename1.backend.Database.open(":memory:"));com.codename1.orm.session.Session s=em.openSession(); + try { + s.createTables(); + org.junit.Assert.assertEquals("LONGTEXT",s.createNativeQuery("SELECT type FROM pragma_table_info('ec_feed_note') WHERE name='note'",String.class).first()); + org.junit.Assert.assertEquals(Long.valueOf(1),s.createNativeQuery("SELECT COUNT(*) FROM pragma_table_info('ec_feed_note') WHERE name='feed_ref'",Long.class).first()); + org.junit.Assert.assertEquals(Long.valueOf(1),s.createNativeQuery("SELECT COUNT(*) FROM pragma_table_info('ec_feed') WHERE name='signal'",Long.class).first()); + org.junit.Assert.assertTrue(s.schemaReport().getIssues().toString(),s.schemaReport().getIssues().isEmpty()); + s.beginTransaction();Object feed=type.newInstance();type.getField("signal").set(feed,"up");((List)type.getField("notes").get(feed)).add("first");s.persist(feed);s.commitTransaction();Object id=type.getField("id").get(feed);s.clear(); + feed=s.find(type,id);org.junit.Assert.assertEquals("up",type.getField("signal").get(feed));org.junit.Assert.assertEquals(1,s.count(feed,"notes"));org.junit.Assert.assertEquals("first",s.createNativeQuery("SELECT note FROM ec_feed_note WHERE feed_ref="+id,String.class).first()); + } finally { s.close();em.close();loader.close(); } + // The owner column named in both places is ambiguous. + File conflict=tmp.newFolder("ecconflict");sources.clear(); + sources.put("ecconflict.Feed","package ecconflict; import com.codename1.annotations.*; import com.codename1.annotations.db.*; @Entity public class Feed { @Id public long id; @ElementCollection(joinColumn=\"a\") @JoinTable(name=\"ec_conflict\",joinColumn=\"b\") public java.util.List notes=new java.util.ArrayList(); }"); + JavaSourceCompiler.compile(sources,conflict,Arrays.asList(testClassesDir())); + ProcessorContext bad=runProcessor(conflict,backendClasspath());assertTrue(bad.hasErrors());assertTrue(bad.getErrors().toString(),bad.getErrors().toString().contains("ElementCollection.joinColumn")); + } + @Test public void jpqlAcceptsSupplementaryJavaIdentifiers() throws Exception { String letter="\ud801\udc00",entityName=letter+"Entity",field=letter+"Value",continuation="tail"+letter; From 7395012da250bb994e88f52439db892a29a788da Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:14:46 +0100 Subject: [PATCH 18/25] Backend: the {sha256} encoder uses the shared Hash.toHex Co-Authored-By: Claude Opus 5.5 --- .../security/crypto/Sha256HexPasswordEncoder.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java b/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java index d4be4a6d85e..827f8c78b8e 100644 --- a/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java +++ b/vm/backend/src/com/codename1/backend/security/crypto/Sha256HexPasswordEncoder.java @@ -23,6 +23,7 @@ package com.codename1.backend.security.crypto; import com.codename1.backend.Crypto; +import com.codename1.security.Hash; import java.io.UnsupportedEncodingException; /// The hexadecimal SHA-256 of a value, unsalted and in one round. Registered as @@ -36,20 +37,12 @@ /// [#matches(CharSequence, String)] compares in constant time and accepts upper /// or lower case hexadecimal, and [#upgradeEncoding(String)] is always true. public final class Sha256HexPasswordEncoder implements PasswordEncoder { - private static final char[] DIGITS = "0123456789abcdef".toCharArray(); - @Override public String encode(CharSequence rawPassword) { if (rawPassword == null) { throw new IllegalArgumentException("rawPassword cannot be null"); } - byte[] digest = Crypto.sha256(utf8(rawPassword.toString())); - char[] out = new char[digest.length * 2]; - for (int iter = 0 ; iter < digest.length ; iter++) { - out[iter * 2] = DIGITS[(digest[iter] >> 4) & 0xf]; - out[iter * 2 + 1] = DIGITS[digest[iter] & 0xf]; - } - return new String(out); + return Hash.toHex(Crypto.sha256(utf8(rawPassword.toString()))); } @Override From 97391aa6ef144a1bd4c2253f118038ba7231f114 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:14:23 +0100 Subject: [PATCH 19/25] Backend: share core com.codename1.xml, delete the backend's XML parser XMLParser, Element, ParserCallback and the new CharEntities and XMLParseException are @SharedWithBackend. XMLParser no longer links HTMLUtils (entity tables moved to CharEntities, HTMLUtils delegates) or Log (I/O errors go to the callback as ERROR_IO). Strict mode now refuses malformed input with XMLParseException, caps nesting (setMaxDepth, default 256), skips DOCTYPE internal subsets without expanding them, and defines only the XML entities. Element gains child, children, childText. PropertyLists uses the shared parser; XmlElement, XmlPullParser and XmlException are deleted. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/parparvm-tests.yml | 2 + .../src/com/codename1/ui/html/HTMLUtils.java | 112 +--- .../src/com/codename1/xml/CharEntities.java | 228 ++++++++ .../src/com/codename1/xml/Element.java | 72 +++ .../src/com/codename1/xml/ParserCallback.java | 3 + .../com/codename1/xml/XMLParseException.java | 34 +- .../src/com/codename1/xml/XMLParser.java | 340 ++++++++++-- maven/backend/spotbugs-exclude.xml | 14 + .../backend/xml/PropertyListsTest.java | 72 ++- .../xml/XMLParserStrictModeTest.java | 103 +++- .../codename1/backend/xml/PropertyLists.java | 139 +++-- .../com/codename1/backend/xml/XmlElement.java | 186 ------- .../codename1/backend/xml/XmlPullParser.java | 516 ------------------ .../codename1/backend/xml/package-info.java | 20 +- 14 files changed, 897 insertions(+), 944 deletions(-) create mode 100644 CodenameOne/src/com/codename1/xml/CharEntities.java rename vm/backend/src/com/codename1/backend/xml/XmlException.java => CodenameOne/src/com/codename1/xml/XMLParseException.java (62%) delete mode 100644 vm/backend/src/com/codename1/backend/xml/XmlElement.java delete mode 100644 vm/backend/src/com/codename1/backend/xml/XmlPullParser.java diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index dd741207f0c..d1b05e4a785 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -48,6 +48,7 @@ on: - 'CodenameOne/src/com/codename1/qrcode/QrCode.java' - 'CodenameOne/src/com/codename1/impl/MdnsEngine.java' - 'CodenameOne/src/com/codename1/util/regex/**' + - 'CodenameOne/src/com/codename1/xml/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' @@ -136,6 +137,7 @@ on: - 'CodenameOne/src/com/codename1/qrcode/QrCode.java' - 'CodenameOne/src/com/codename1/impl/MdnsEngine.java' - 'CodenameOne/src/com/codename1/util/regex/**' + - 'CodenameOne/src/com/codename1/xml/**' - 'CodenameOne/src/com/codename1/orm/session/**' - 'CodenameOne/src/com/codename1/orm/repository/**' - 'CodenameOne/src/com/codename1/annotations/db/**' diff --git a/CodenameOne/src/com/codename1/ui/html/HTMLUtils.java b/CodenameOne/src/com/codename1/ui/html/HTMLUtils.java index 3697bb1ac19..4c0d0f4bba9 100644 --- a/CodenameOne/src/com/codename1/ui/html/HTMLUtils.java +++ b/CodenameOne/src/com/codename1/ui/html/HTMLUtils.java @@ -23,6 +23,7 @@ */ package com.codename1.ui.html; +import com.codename1.xml.CharEntities; import java.util.Hashtable; /// This class contains several useful static methods for HTML @@ -32,52 +33,6 @@ /// the HTML package is no longer used or maintained and may be removed in a future revision public final class HTMLUtils { - /// The char entities strings supported in XML. When a char entity is found these will be compared against first. - private static final String[] XML_CHAR_ENTITIES = { - "lt", // lesser-than - "gt", // greater-than - "amp", // ampersand - "quot", //quotation mark - "apos", // apostrophe - //"bull", //bullet - //"euro" //euro - }; - /// The numericals value of char entities strings above. - private static final int[] XML_CHAR_ENTITIES_VALS = { - 60, // "lt", // lesser-than - 62, // "gt", // greater-than - 38, // "amp", // ampersand - 34, // "quot", //quotation mark - 39, // "apos", // apostrophe - //8226, // "bull", //bullet - //8364 // "euro"}; //euro - }; - /// This is a list of ISO 8859-1 Symbols that can be used as HTML char entities - private static final String[] HTML_BASIC_CHAR_ENTITY_STRINGS = { - "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect", "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr", "deg", "plusmn", "sup2", "sup3", "acute", - "micro", "para", "middot", "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest", "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", - "Ccedil", "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml", "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times", "Oslash", - "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig", "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil", "egrave", "eacute", "ecirc", - "euml", "igrave", "iacute", "icirc", "iuml", "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide", "oslash", "ugrave", "uacute", "ucirc", "uuml", - "yacute", "thorn", "yuml"}; - /// Common HTML 4 named entities outside the contiguous ISO-8859-1 range above. Keeping these in - /// the shared parser utility lets all tolerant HTMLParser clients decode modern pasted fragments - /// without maintaining their own entity tables. - private static final String[] HTML_EXTENDED_CHAR_ENTITY_STRINGS = { - "OElig", "oelig", "Scaron", "scaron", "Yuml", "fnof", "circ", "tilde", - "ensp", "emsp", "thinsp", "zwnj", "zwj", "lrm", "rlm", "ndash", "mdash", - "lsquo", "rsquo", "sbquo", "ldquo", "rdquo", "bdquo", "dagger", "Dagger", "bull", - "hellip", "permil", "prime", "Prime", "lsaquo", "rsaquo", "oline", "frasl", "euro", - "image", "weierp", "real", "trade", "alefsym" - }; - private static final int[] HTML_EXTENDED_CHAR_ENTITY_VALUES = { - 338, 339, 352, 353, 376, 402, 710, 732, - 8194, 8195, 8201, 8204, 8205, 8206, 8207, 8211, 8212, - 8216, 8217, 8218, 8220, 8221, 8222, 8224, 8225, 8226, - 8230, 8240, 8242, 8243, 8249, 8250, 8254, 8260, 8364, - 8465, 8472, 8476, 8482, 8501 - }; - // Prevents instantiation - this class has static method only private HTMLUtils() { @@ -126,71 +81,8 @@ public static String convertHTMLCharEntity(String charEntity) { /// #### Returns /// /// A string containing a single char, or the original char entity string (with & and ;) if the char entity couldn't be resolved - @SuppressWarnings("PMD.EmptyCatchBlock") public static String convertCharEntity(String charEntity, boolean lookupHTMLentities, Hashtable userDefinedCharEntities) { - int charCode = -1; - if (charEntity.startsWith("#")) { //numbered char entity - if (charEntity.startsWith("#x")) { //hex - try { - charCode = Integer.parseInt(charEntity.substring(2), 16); - } catch (NumberFormatException nfe) { - //if not a number - simply ignore char entity - } - } else { - try { - charCode = Integer.parseInt(charEntity.substring(1)); - } catch (NumberFormatException nfe) { - //if not a number - simply ignore char entity - } - } - } else { //not numbered, rather a symbol - charCode = getCharEntityCode(charEntity, lookupHTMLentities, userDefinedCharEntities); - } - - if (charCode != -1) { - return "" + (char) charCode; - } else { - throw new IllegalArgumentException("Unknown character entity - " + charEntity); - //notifyError(ParserCallback.ERROR_UNRECOGNIZED_CHAR_ENTITY,null,null,null, "Unrecognized char entity: "+charEntity); - //return "&"+charEntity+";"; // Another option is to return an empty string, but returning the entity will unravel bugs and will also allow ignoring common mistakes such as using the & char (instead of ') - } - - } - - private static int getCharEntityCode(String symbol, boolean html, Hashtable userDefined) { - // First tries the XML basic char entities - int val = -1; - for (int i = 0; i < XML_CHAR_ENTITIES.length; i++) { - if (symbol.equalsIgnoreCase(XML_CHAR_ENTITIES[i])) { - return XML_CHAR_ENTITIES_VALS[i]; - } - } - if (val != -1) { - return val; - } else { - if (html) { - // Not one of the most popular char codes, proceed to check the ISO-8859-1 symbols array - val = getStringVal(symbol, HTML_BASIC_CHAR_ENTITY_STRINGS); - if (val != -1) { - return val + 160; - } - val = getStringVal(symbol, HTML_EXTENDED_CHAR_ENTITY_STRINGS); - if (val != -1) { - return HTML_EXTENDED_CHAR_ENTITY_VALUES[val]; - } - } - - // Not found in the standard symbol table, see if it is in the user defined symbols table - if (userDefined != null) { - Object charObj = userDefined.get(symbol); - if (charObj != null) { - return ((Integer) charObj).intValue(); - } - } - - // Not found anywhere - return -1; - } + return CharEntities.convert(charEntity, lookupHTMLentities, userDefinedCharEntities); } /// Encodes the specified string to "percent-encoding" or URL encoding. diff --git a/CodenameOne/src/com/codename1/xml/CharEntities.java b/CodenameOne/src/com/codename1/xml/CharEntities.java new file mode 100644 index 00000000000..1a3ff150aec --- /dev/null +++ b/CodenameOne/src/com/codename1/xml/CharEntities.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.xml; + +import java.util.Hashtable; + +/// The character entity tables of the XML and HTML parsers, and the lookups +/// over them. +/// +/// It lives here rather than in `com.codename1.ui.html` so that [XMLParser] can +/// decode entities without linking any UI class: the parser is shared with the +/// server runtime. `com.codename1.ui.html.HTMLUtils` delegates to it, so the +/// lenient HTML behaviour is unchanged. +@com.codename1.impl.SharedWithBackend +public final class CharEntities { + + /// The char entities strings supported in XML. When a char entity is found these will be compared against first. + private static final String[] XML_CHAR_ENTITIES = { + "lt", // lesser-than + "gt", // greater-than + "amp", // ampersand + "quot", //quotation mark + "apos", // apostrophe + }; + /// The numericals value of char entities strings above. + private static final int[] XML_CHAR_ENTITIES_VALS = { + 60, // "lt", // lesser-than + 62, // "gt", // greater-than + 38, // "amp", // ampersand + 34, // "quot", //quotation mark + 39, // "apos", // apostrophe + }; + /// This is a list of ISO 8859-1 Symbols that can be used as HTML char entities + private static final String[] HTML_BASIC_CHAR_ENTITY_STRINGS = { + "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect", "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr", "deg", "plusmn", "sup2", "sup3", "acute", + "micro", "para", "middot", "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest", "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", + "Ccedil", "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml", "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times", "Oslash", + "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig", "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil", "egrave", "eacute", "ecirc", + "euml", "igrave", "iacute", "icirc", "iuml", "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide", "oslash", "ugrave", "uacute", "ucirc", "uuml", + "yacute", "thorn", "yuml"}; + /// Common HTML 4 named entities outside the contiguous ISO-8859-1 range above. Keeping these in + /// the shared parser utility lets all tolerant HTMLParser clients decode modern pasted fragments + /// without maintaining their own entity tables. + private static final String[] HTML_EXTENDED_CHAR_ENTITY_STRINGS = { + "OElig", "oelig", "Scaron", "scaron", "Yuml", "fnof", "circ", "tilde", + "ensp", "emsp", "thinsp", "zwnj", "zwj", "lrm", "rlm", "ndash", "mdash", + "lsquo", "rsquo", "sbquo", "ldquo", "rdquo", "bdquo", "dagger", "Dagger", "bull", + "hellip", "permil", "prime", "Prime", "lsaquo", "rsaquo", "oline", "frasl", "euro", + "image", "weierp", "real", "trade", "alefsym" + }; + private static final int[] HTML_EXTENDED_CHAR_ENTITY_VALUES = { + 338, 339, 352, 353, 376, 402, 710, 732, + 8194, 8195, 8201, 8204, 8205, 8206, 8207, 8211, 8212, + 8216, 8217, 8218, 8220, 8221, 8222, 8224, 8225, 8226, + 8230, 8240, 8242, 8243, 8249, 8250, 8254, 8260, 8364, + 8465, 8472, 8476, 8482, 8501 + }; + + private CharEntities() { + } + + /// Converts a char entity to the matching character, the lenient way the + /// HTML parser always has: the XML names match ignoring case, and a numeric + /// reference is truncated to one `char`. + /// + /// #### Parameters + /// + /// - `charEntity`: The char entity to convert (Not including the & and ;) + /// + /// - `lookupHTMLentities`: true to include the basic HTML named char entities (unicode 160-255), false otherwise + /// + /// - `userDefinedCharEntities`: A hashtable containing (String,int) denoting the char entity name and its unicode + /// + /// #### Returns + /// + /// A string containing a single char + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: if the entity is unknown + @SuppressWarnings("PMD.EmptyCatchBlock") + public static String convert(String charEntity, boolean lookupHTMLentities, Hashtable userDefinedCharEntities) { + int charCode = -1; + if (charEntity.startsWith("#")) { //numbered char entity + if (charEntity.startsWith("#x")) { //hex + try { + charCode = Integer.parseInt(charEntity.substring(2), 16); + } catch (NumberFormatException nfe) { + //if not a number - simply ignore char entity + } + } else { + try { + charCode = Integer.parseInt(charEntity.substring(1)); + } catch (NumberFormatException nfe) { + //if not a number - simply ignore char entity + } + } + } else { //not numbered, rather a symbol + charCode = getCharEntityCode(charEntity, lookupHTMLentities, userDefinedCharEntities); + } + + if (charCode != -1) { + return String.valueOf((char) charCode); + } + throw new IllegalArgumentException("Unknown character entity - " + charEntity); + } + + /// Decodes a reference the way XML defines it: the five predefined names, + /// matched exactly, a decimal or hexadecimal character reference to a legal + /// code point (a supplementary one becomes a surrogate pair), or a name in + /// `userDefinedCharEntities`. Entities a document declares itself are never + /// consulted, so nothing a document says can be expanded. + /// + /// #### Parameters + /// + /// - `ref`: the reference without the & and ; + /// + /// - `userDefinedCharEntities`: names the application added, or null + /// + /// #### Returns + /// + /// the decoded characters, or null when the reference is undefined or invalid + public static String decodeXml(String ref, Hashtable userDefinedCharEntities) { + for (int i = 0; i < XML_CHAR_ENTITIES.length; i++) { + if (XML_CHAR_ENTITIES[i].equals(ref)) { + return String.valueOf((char) XML_CHAR_ENTITIES_VALS[i]); + } + } + if (ref.length() > 1 && ref.charAt(0) == '#') { + int code = parseCharRef(ref); + if (code <= 0 || (code >= 0xd800 && code <= 0xdfff)) { + return null; + } + if (code >= 0x10000) { + int v = code - 0x10000; + char[] pair = {(char) (0xd800 + (v >> 10)), (char) (0xdc00 + (v & 0x3ff))}; + return new String(pair); + } + return String.valueOf((char) code); + } + if (userDefinedCharEntities != null) { + Object code = userDefinedCharEntities.get(ref); + if (code instanceof Integer) { + return String.valueOf((char) ((Integer) code).intValue()); + } + } + return null; + } + + /// The code point of `#123` or `#x7b`, or -1 when it is not one. + private static int parseCharRef(String ref) { + boolean hex = ref.length() > 2 && (ref.charAt(1) == 'x' || ref.charAt(1) == 'X'); + int start = hex ? 2 : 1; + int radix = hex ? 16 : 10; + int value = 0; + for (int i = start; i < ref.length(); i++) { + int digit = Character.digit(ref.charAt(i), radix); + if (digit < 0) { + return -1; + } + value = value * radix + digit; + if (value > 0x10ffff) { + return -1; + } + } + return value; + } + + private static int getCharEntityCode(String symbol, boolean html, Hashtable userDefined) { + // First tries the XML basic char entities + for (int i = 0; i < XML_CHAR_ENTITIES.length; i++) { + if (symbol.equalsIgnoreCase(XML_CHAR_ENTITIES[i])) { + return XML_CHAR_ENTITIES_VALS[i]; + } + } + if (html) { + // Not one of the most popular char codes, proceed to check the ISO-8859-1 symbols array + int val = indexOf(symbol, HTML_BASIC_CHAR_ENTITY_STRINGS); + if (val != -1) { + return val + 160; + } + val = indexOf(symbol, HTML_EXTENDED_CHAR_ENTITY_STRINGS); + if (val != -1) { + return HTML_EXTENDED_CHAR_ENTITY_VALUES[val]; + } + } + + // Not found in the standard symbol table, see if it is in the user defined symbols table + if (userDefined != null) { + Object charObj = userDefined.get(symbol); + if (charObj instanceof Integer) { + return ((Integer) charObj).intValue(); + } + } + + // Not found anywhere + return -1; + } + + private static int indexOf(String str, String[] options) { + for (int i = 0; i < options.length; i++) { + if (str.equals(options[i])) { + return i; + } + } + return -1; + } +} diff --git a/CodenameOne/src/com/codename1/xml/Element.java b/CodenameOne/src/com/codename1/xml/Element.java index ea161e73cb1..e3f61a72b6e 100644 --- a/CodenameOne/src/com/codename1/xml/Element.java +++ b/CodenameOne/src/com/codename1/xml/Element.java @@ -27,10 +27,12 @@ import java.util.Enumeration; import java.util.Hashtable; import java.util.Iterator; +import java.util.List; import java.util.Vector; /// The Element class defines a single XML element with its attributes and children. /// Due to its hierarchial nature, this class can be used for a single "leaf" Element, for more complex elements (with child elements), and up to describing the entire document. +@com.codename1.impl.SharedWithBackend public class Element implements Iterable { /// A constant that can be used for the get descendants methods to denote infinite recursion @@ -950,4 +952,74 @@ public boolean isEmpty() { public Iterator iterator() { return children.iterator(); } + + /// Returns the first child element with the given tag name, the same as + /// `getFirstChildByTagName(String)`. + /// + /// #### Parameters + /// + /// - `name`: The child's tag name + /// + /// #### Returns + /// + /// the first child with the specified name, or null if there is none + public Element child(String name) { + return getFirstChildByTagName(name); + } + + /// Returns the child elements with the given tag name, in document order. + /// Unlike `getChildrenByTagName(String)` this answers a typed list. + /// + /// #### Parameters + /// + /// - `name`: The tag name to look for + /// + /// #### Returns + /// + /// the matching children; empty, never null, when there are none + public List children(String name) { + List out = new ArrayList(); + if (children == null) { + return out; + } + int count = children.size(); + for (int i = 0; i < count; i++) { + Element child = children.get(i); + if (!child.textElement && cmpTagName(name, child)) { + out.add(child); + } + } + return out; + } + + /// Returns the text of the first child element with the given tag name: + /// its text children joined, CDATA included, with the leading and trailing + /// whitespace trimmed. + /// + /// #### Parameters + /// + /// - `name`: The child's tag name + /// + /// #### Returns + /// + /// the child's trimmed text, empty when it has none, or null if there is + /// no such child + public String childText(String name) { + Element child = getFirstChildByTagName(name); + if (child == null) { + return null; + } + if (child.children == null) { + return ""; + } + StringBuilder b = new StringBuilder(); + int count = child.children.size(); + for (int i = 0; i < count; i++) { + Element c = child.children.get(i); + if (c.textElement) { + b.append(c.name); + } + } + return b.toString().trim(); + } } diff --git a/CodenameOne/src/com/codename1/xml/ParserCallback.java b/CodenameOne/src/com/codename1/xml/ParserCallback.java index a3c3bfa914a..e1ceea6cd1c 100644 --- a/CodenameOne/src/com/codename1/xml/ParserCallback.java +++ b/CodenameOne/src/com/codename1/xml/ParserCallback.java @@ -24,6 +24,7 @@ package com.codename1.xml; /// A callback used to dispatch errors encountered while parsing XML resources +@com.codename1.impl.SharedWithBackend public interface ParserCallback { /// Error code denoting that an unsupported tag was found in the XML @@ -56,6 +57,8 @@ public interface ParserCallback { /// Error code denoting that the encoding the page needed according to its charset (usually specified in the content-type response header) is unsupported in the device int ERROR_ENCODING = 9; + /// Error code denoting that reading the document failed with an `IOException` + int ERROR_IO = 10; /// Called when encountering an error while parsing the XML document. /// When implementing this, the developer should return true if the error should be ignored and the document needs to be further parsed, or false to stop parsing and issue an error to the user diff --git a/vm/backend/src/com/codename1/backend/xml/XmlException.java b/CodenameOne/src/com/codename1/xml/XMLParseException.java similarity index 62% rename from vm/backend/src/com/codename1/backend/xml/XmlException.java rename to CodenameOne/src/com/codename1/xml/XMLParseException.java index 00d60339cdf..960e1110e67 100644 --- a/vm/backend/src/com/codename1/backend/xml/XmlException.java +++ b/CodenameOne/src/com/codename1/xml/XMLParseException.java @@ -20,26 +20,24 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.backend.xml; +package com.codename1.xml; -import java.io.IOException; - -/// Thrown when a document is not well-formed XML, uses a construct this parser -/// refuses (an entity it does not know, nesting past the limit), or is not the -/// structure a reader such as [PropertyLists] expects. +/// Thrown by an [XMLParser] in strict mode (see [XMLParser#setStrict(boolean)]) +/// when the document is not well formed, nests deeper than +/// [XMLParser#setMaxDepth(int)], or references an entity the parser does not +/// define. /// -/// The message names the problem and, where one is known, the character offset -/// in the decoded document. -public class XmlException extends IOException { - private static final long serialVersionUID = 1L; - - /// An exception with the given message. - public XmlException(String message) { +/// It extends `IllegalArgumentException`, which is what the parser has always +/// thrown when a [ParserCallback] asks it to stop, so code that catches that +/// keeps working. +@com.codename1.impl.SharedWithBackend +public class XMLParseException extends IllegalArgumentException { + /// Creates the exception with the description of what is wrong. + /// + /// #### Parameters + /// + /// - `message`: the description + public XMLParseException(String message) { super(message); } - - /// An exception with the given message, caused by `cause`. - public XmlException(String message, Throwable cause) { - super(message, cause); - } } diff --git a/CodenameOne/src/com/codename1/xml/XMLParser.java b/CodenameOne/src/com/codename1/xml/XMLParser.java index e8b53f90deb..ef76447e5fe 100644 --- a/CodenameOne/src/com/codename1/xml/XMLParser.java +++ b/CodenameOne/src/com/codename1/xml/XMLParser.java @@ -23,12 +23,11 @@ */ package com.codename1.xml; -import com.codename1.io.Log; -import com.codename1.ui.html.HTMLUtils; - import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.Hashtable; +import java.util.List; /// Parser class used to parse an XML document into a DOM object (Element). This code was originally /// developed to parse HTML and as a result isn't as strict as most XML parsers and can parse many HTML documents @@ -53,6 +52,7 @@ /// Vector result = new Vector(); /// Element e = (Element)parent; /// for(int iter = 0 ; iter +@com.codename1.impl.SharedWithBackend public class XMLParser { private static final Element END_TAG = new Element(); /// A constant containing the CDATA tag identifier (minus the C which is read anyway first) @@ -94,6 +94,15 @@ public class XMLParser { private String[] pool; private char[][] poolChars; private int poolCount; + // Strict mode only: the names of the elements currently open, the cap on + // how many may be, whether the root element has been read, and the + // attribute names of the tag being read (to refuse a duplicate). + private static final int DEFAULT_MAX_DEPTH = 256; + private int maxDepth = DEFAULT_MAX_DEPTH; + private String[] openNames = new String[16]; + private int openCount; + private boolean rootSeen; + private final List tagAttributes = new ArrayList(); /// The char entities strings supported in XML. When a char entity is found these will be compared against first. /// @@ -146,6 +155,8 @@ private void resetBuffer() { buffOffset = 0; buffSize = -1; contentDepth = 0; + openCount = 0; + rootSeen = false; if (pool != null) { for (int i = 0; i < POOL_SIZE; i++) { pool[i] = null; @@ -404,7 +415,7 @@ public void addCharEntitiesRange(String[] symbols, int startcode) { /// A string containing a single char, or the original char entity string (with & and ;) if the char entity couldn't be resolved protected String convertCharEntity(String charEntity) { try { - return HTMLUtils.convertCharEntity(charEntity, false, userDefinedCharEntities); + return CharEntities.convert(charEntity, false, userDefinedCharEntities); } catch (IllegalArgumentException iae) { notifyError(ParserCallback.ERROR_UNRECOGNIZED_CHAR_ENTITY, null, null, null, "Unrecognized char entity: " + charEntity); // Another option is to return an empty string, but returning the entity will unravel bugs and will also allow ignoring common mistakes such as using the & char (instead of ') @@ -456,7 +467,9 @@ public Element parse(Reader is) { try { parseTagContent(rootElement, is); } catch (IOException ioe) { - Log.e(ioe); + // Reported through the callback rather than a log: the parser is + // shared with the server runtime, which has no Codename One Log. + notifyError(ParserCallback.ERROR_IO, null, null, null, "I/O error while parsing: " + ioe.getMessage()); } if (rootElement.getNumChildren() == 0) { notifyError(ParserCallback.ERROR_NO_ROOTS, null, null, null, "XML document contains no root element."); @@ -539,6 +552,9 @@ public void eventParser(Reader r) throws IOException { resetBuffer(); eventParser = true; parseTagContent(null, r); + if (strict && !rootSeen) { + throw new XMLParseException("no root element"); + } } /// Invoked when the event parser encounters a text element. @@ -636,7 +652,8 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException while (val != -1) { if (c == '<') { - if ((includeWhitespacesBetweenTags) && (leadingSpace) && (!inText) && (element != null) && (element.getNumChildren() > 0)) { + if ((includeWhitespacesBetweenTags) && (leadingSpace) && (!inText) && (element != null) && (element.getNumChildren() > 0) + && !(strict && contentDepth == 1)) { leadingSpace = false; text.clear(); text.add(' '); @@ -664,6 +681,10 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException Element childElement = parseTag(is); + if (childElement == END_TAG && strict) { + strictEndTag(is); + return; + } if (childElement == END_TAG) { //was actually an ending tag CharBuf closingTag = nameBuf; @@ -723,8 +744,12 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException inEntity = false; } } else if (c == '&') { //start char entity - charEntity.clear(); - inEntity = true; + if (strict) { + readStrictReference(is, text); + } else { + charEntity.clear(); + inEntity = true; + } } else { text.add(c); // Plain text runs to the next markup or entity; copy the @@ -746,6 +771,11 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException buffOffset = end; } } + } else if (strict && contentDepth == 1 && (ws(c) || c == (char) 0xfeff)) { + // whitespace, or a byte order mark, around the root element + leadingSpace = true; + } else if (strict && contentDepth == 1) { + throw new XMLParseException("text outside the root element"); } else if (keepAllText || !isWhiteSpace(c)) { text.clear(); inText = true; @@ -755,8 +785,12 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException } if (c == '&') { //text starts with a character entity (i.e.  ) // The & is not included in the string we accumulate - charEntity.clear(); - inEntity = true; + if (strict) { + readStrictReference(is, text); + } else { + charEntity.clear(); + inEntity = true; + } } else { // re-dispatch through the plain text branch so the bulk copy applies continue; @@ -767,6 +801,159 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException val = read(is); c = (char) val; } + if (strict && contentDepth > 1) { + throw new XMLParseException("unexpected end of document inside <" + openName() + ">"); + } + } + + private String openName() { + return openCount > 0 ? openNames[openCount - 1] : "?"; + } + + /// Strict mode: reads the rest of an end tag, whose `') { + if (i < 0) { + throw new XMLParseException("unterminated end tag "); + } + closingTag.add((char) i); + i = read(is); + } + while (closingTag.len > 0 && ws(closingTag.chars[closingTag.len - 1])) { + closingTag.len--; + } + String ct = name(closingTag); + if (contentDepth < 2 || openCount == 0) { + throw new XMLParseException("end tag with no open element"); + } + String open = openNames[openCount - 1]; + if (!ct.equals(open)) { + throw new XMLParseException("end tag does not match <" + open + ">"); + } + if (eventParser) { + endTag(ct); + } + } + + /// Strict mode: reads an entity reference whose `&` is already consumed + /// and appends what it stands for. Only the five XML names, character + /// references and names added with [#addCharEntity(String, int)] are + /// defined; a document type declaration never defines one, so nothing a + /// document declares is ever expanded. + private void readStrictReference(Reader is, CharBuf out) throws IOException { + CharBuf ref = entityBuf; + ref.clear(); + while (true) { + int i = read(is); + if (i == ';') { + break; + } + if (i < 0 || ref.len >= 32 || !isLegalStrictReferenceCharacter((char) i)) { + throw new XMLParseException("unterminated entity reference &" + ref); + } + ref.add((char) i); + } + String name = ref.toString(); + String decoded = CharEntities.decodeXml(name, userDefinedCharEntities); + if (decoded == null) { + throw new XMLParseException("undefined entity &" + name + ";"); + } + out.add(decoded); + } + + private static boolean isLegalStrictReferenceCharacter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || c == '#' || c == '_' || c == '-' || c == '.' || c == ':'; + } + + /// Reads one character of a tag. In strict mode the end of the document + /// inside a tag is an error; otherwise -1 is handed on as a char, as the + /// lenient parser always has. + private char tagChar(Reader is) throws IOException { + int i = read(is); + if (i < 0 && strict) { + throw new XMLParseException("unexpected end of document inside a tag"); + } + return (char) i; + } + + /// What `parseTag` answers when the document ends inside a tag. + private Element eofInTag() { + if (strict) { + throw new XMLParseException("unexpected end of document inside a tag"); + } + return END_TAG; + } + + /// Parses the content of the element `tn` just opened, keeping strict + /// mode's stack of open names and its depth cap. + private void tagContent(Element element, Reader is, String tn) throws IOException { + if (!strict) { + parseTagContent(element, is); + return; + } + if (openCount == openNames.length) { + String[] a = new String[openCount * 2]; + System.arraycopy(openNames, 0, a, 0, openCount); + openNames = a; + } + openNames[openCount++] = tn; + try { + parseTagContent(element, is); + } finally { + openCount--; + openNames[openCount] = null; + } + } + + /// Strict mode: skips a `` or other ` 0) { + pending--; + } + if (i < 0 || i == 0xffff) { + throw new XMLParseException("unterminated declaration"); + } + char ch = (char) i; + if (quote != 0) { + if (ch == quote) { + quote = 0; + } + } else if (ch == '"' || ch == '\'') { + quote = ch; + } else if (ch == '[') { + brackets++; + } else if (ch == ']') { + brackets--; + } else if (ch == '>' && brackets <= 0) { + break; + } + } + if (eventParser) { + return null; + } + Element declaration = createNewElement("XML declaration"); + declaration.isComment = true; + return declaration; } /// Checks if the specified character is a white space or not. @@ -807,27 +994,35 @@ protected Element parseTag(Reader is) throws IOException { final boolean fold = !eventParser && !caseSensitive && !strict; //boolean procInst=false; // Support for the styleshhet processing instruction was removed, as it is not supported in most browsers, and it causes problems by adding tags before the HTML element (Makign the document with multiple roots) - char c = (char) read(is); + char c = tagChar(is); if (c == '/') { return END_TAG; //end tag } else if (c == '!') { - c = (char) read(is); - char c2 = (char) read(is); + c = tagChar(is); + char c2 = tagChar(is); if ((c == '-') && (c2 == '-')) { //comment return parseCommentOrXMLDeclaration(is, "-->"); + } else if (strict && !((c == '[') && (c2 == 'C'))) { + return skipDeclaration(is, c, c2); } else if ((c == '[') && (c2 == 'C')) { // CDATA? - c = (char) read(is); + c = tagChar(is); int idx = 0; int clen = CDATA_STR.length(); while ((idx < clen) && (c == CDATA_STR.charAt(idx))) { idx++; if (idx < clen) { - c = (char) read(is); + c = tagChar(is); } } if (idx == clen) { //found CDATA + if (strict && contentDepth == 1) { + throw new XMLParseException("text outside the root element"); + } return parseCommentOrXMLDeclaration(is, "]]>"); //parse doctypes i.e. as comments as well - i.e. ignore them } else { // some other unknown tag + if (strict) { + throw new XMLParseException("malformed CDATA section"); + } return parseCommentOrXMLDeclaration(is, ">"); //parse doctypes i.e. as comments as well - i.e. ignore them } @@ -840,7 +1035,7 @@ protected Element parseTag(Reader is) throws IOException { //read and ignore any whitespaces before tag name while (ws(c)) { - c = (char) read(is); + c = tagChar(is); } //collect tag name @@ -853,18 +1048,33 @@ protected Element parseTag(Reader is) throws IOException { // Fix for http://code.google.com/p/codenameone/issues/detail?id=900 if (i == -1) { - return END_TAG; + return eofInTag(); } c = (char) i; } //read and ignore any whitespaces after tag name while (ws(c)) { - c = (char) read(is); + c = tagChar(is); } boolean processTag = true; String tn = name(tagName); + if (strict) { + if (tagName.len == 0) { + throw new XMLParseException("expected a tag name after <"); + } + if (contentDepth == 1) { + if (rootSeen) { + throw new XMLParseException("a second root element <" + tn + ">"); + } + rootSeen = true; + } + if (openCount >= maxDepth) { + throw new XMLParseException("elements nested deeper than " + maxDepth); + } + tagAttributes.clear(); + } if (eventParser) { processTag = startTag(tn); } @@ -920,11 +1130,11 @@ protected Element parseTag(Reader is) throws IOException { if (c == '>') { //tag declartion ended, process content if (!isEmptyTag(tn)) { - parseTagContent(element, is); + tagContent(element, is, tn); } return element; } else if (c == '/') { //closed tag - no content - c = (char) read(is); + c = tagChar(is); if (c == '>') { if (eventParser) { endTag(tn); @@ -939,33 +1149,33 @@ protected Element parseTag(Reader is) throws IOException { while (true) { curAttribute.clear(); curAttribute.add(c); - c = (char) read(is); + c = tagChar(is); while ((!ws(c)) && (c != '=') && (c != '>')) { if (fold) { c = Character.toLowerCase(c); } curAttribute.add(c); - c = (char) read(is); + c = tagChar(is); } if (c == '>') { // tag close char shouldn't be found here, but if the XML is slightly malformed we return the element notifyError(ParserCallback.ERROR_UNEXPECTED_TAG_CLOSING, tn, curAttribute.toString(), null, "Unexpected tag closing in tag " + tn + ", attribute=" + curAttribute); if (!isEmptyTag(tn)) { - parseTagContent(element, is); + tagContent(element, is, tn); } return element; } //read and ignore any whitespaces after attribute name while (ws(c)) { - c = (char) read(is); + c = tagChar(is); } if (c != '=') { notifyError(ParserCallback.ERROR_UNEXPECTED_CHARACTER, tn, curAttribute.toString(), null, "Unexpected character " + c + ", expected '=' after attribute " + curAttribute + " in tag " + tn); if (c == '>') { // tag close char shouldn't be found here, but if the XML is slightly malformed we return the element if (!isEmptyTag(tn)) { - parseTagContent(element, is); + tagContent(element, is, tn); } return element; } @@ -974,10 +1184,10 @@ protected Element parseTag(Reader is) throws IOException { continue; //if attribute is not followed by = then process the next attribute } - c = (char) read(is); + c = tagChar(is); //read and ignore any whitespaces before attribute value while (ws(c)) { - c = (char) read(is); + c = tagChar(is); } char quote = ' '; @@ -985,6 +1195,8 @@ protected Element parseTag(Reader is) throws IOException { if ((c == '"') || (c == '\'')) { quote = c; + } else if (strict) { + throw new XMLParseException("attribute " + curAttribute + " of <" + tn + "> is not quoted"); } else { curValue.add(c); } @@ -995,18 +1207,22 @@ protected Element parseTag(Reader is) throws IOException { while (!ended) { int i = read(is); if (i < 0) { - return END_TAG; + return eofInTag(); } c = (char) i; if (c == quote) { ended = true; i = read(is); if (i < 0) { - return END_TAG; + return eofInTag(); } c = (char) i; } else if ((quote == ' ') && ((c == '/') || (c == '>') || (ws(c)))) { ended = true; + } else if (strict && c == '<') { + throw new XMLParseException("< in the value of attribute " + curAttribute + " of <" + tn + ">"); + } else if (strict && c == '&') { + readStrictReference(is, curValue); } else if (c == '&') { if (inEntity) { curValue.add('&'); // Wasn't a char entit, probably a url as a parameter : i.e. param="/test?p=val&pw=val2&p3=val3 @@ -1040,6 +1256,15 @@ protected Element parseTag(Reader is) throws IOException { String an = attributeName(curAttribute); String av = value(curValue); + if (strict) { + if (tagAttributes.contains(an)) { + throw new XMLParseException("duplicate attribute " + an + " in <" + tn + ">"); + } + tagAttributes.add(an); + if (!ws(c) && c != '>' && c != '/') { + throw new XMLParseException("expected whitespace after attribute " + an + " of <" + tn + ">"); + } + } if (eventParser) { attribute(tn, an, av); } else { @@ -1056,20 +1281,20 @@ protected Element parseTag(Reader is) throws IOException { while (ws(c)) { int i = read(is); if (i < 0) { - return END_TAG; + return eofInTag(); } c = (char) i; } if (c == '>') { //tag declartion ended, process content if (!isEmptyTag(tn)) { - parseTagContent(element, is); + tagContent(element, is, tn); } return element; } else if (c == '/') { //closed tag - no content int i = read(is); if (i < 0) { - return END_TAG; + return eofInTag(); } c = (char) i; if (c == '>') { @@ -1113,6 +1338,10 @@ protected Element parseCommentOrXMLDeclaration(Reader is, String endTag) throws while (!ended) { int in = read(is); if (in == -1) { + if (strict) { + throw new XMLParseException("unterminated " + ("-->".equals(endTag) ? "comment" + : "]]>".equals(endTag) ? "CDATA section" : "declaration")); + } // input stream ended abruptly break; } @@ -1198,6 +1427,9 @@ protected void notifyError(int errorId, String tag, String attribute, String val throw new IllegalArgumentException(description); } } + if (strict) { + throw new XMLParseException(description); + } } /// Returns true if this element is supported, false otherwise @@ -1270,7 +1502,25 @@ public void setCaseSensitive(boolean caseSensitive) { /// ` ` keeps its space; /// /// - whitespace is the four XML whitespace characters and `isWhiteSpace` - /// is not consulted. + /// is not consulted; + /// + /// - a document that is not well formed is refused with an + /// [XMLParseException]: a mismatched, stray or missing end tag, a second + /// root element or text outside the root, an unquoted or duplicate + /// attribute, a `<` in an attribute value, an unterminated comment, CDATA + /// section or declaration, and an undefined or malformed entity + /// reference. Every error reported to a [ParserCallback] is thrown, even + /// when the callback asks to continue; + /// + /// - only the five XML entities, character references and names added with + /// [#addCharEntity(String, int)] are defined. A document type + /// declaration is skipped, internal subset included, and never + /// interpreted, so no entity it declares is expanded and no external one + /// is fetched: the parser is safe from external entity and entity + /// expansion attacks; + /// + /// - elements nest at most [#setMaxDepth(int)] deep (256 by default), so a + /// hostile document cannot overflow the stack of the recursive parser. /// /// #### Parameters /// @@ -1288,6 +1538,30 @@ public boolean isStrict() { return strict; } + /// Sets the deepest element nesting strict mode accepts; a deeper document + /// is an [XMLParseException]. The default is 256. The lenient mode does + /// not check nesting. + /// + /// #### Parameters + /// + /// - `maxDepth`: the deepest nesting accepted, at least 1 + public void setMaxDepth(int maxDepth) { + if (maxDepth < 1) { + throw new IllegalArgumentException("maxDepth must be positive"); + } + this.maxDepth = maxDepth; + } + + /// Returns the deepest element nesting strict mode accepts, see + /// `setMaxDepth(int)`. + /// + /// #### Returns + /// + /// the nesting cap + public int getMaxDepth() { + return maxDepth; + } + /// When on, the namespace prefix is removed from tag and attribute names as /// they are read, so `w:p` is reported as `p` and `w:val` as `val`. Off by /// default because it discards information: `w:t` and `m:t` both become diff --git a/maven/backend/spotbugs-exclude.xml b/maven/backend/spotbugs-exclude.xml index 375269ad05c..a823e1e4db4 100644 --- a/maven/backend/spotbugs-exclude.xml +++ b/maven/backend/spotbugs-exclude.xml @@ -182,4 +182,18 @@ + + + + + + + + + + diff --git a/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java index bb8316725d4..371ec38da54 100644 --- a/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/xml/PropertyListsTest.java @@ -22,8 +22,12 @@ */ package com.codename1.backend.xml; +import com.codename1.xml.Element; +import com.codename1.xml.XMLParseException; +import com.codename1.xml.XMLParser; import org.junit.jupiter.api.Test; +import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.List; @@ -93,7 +97,7 @@ void findsThePlistInsideASignedProfile() throws Exception { void externalEntityIsNeverExpanded() { String xxe = "" + "\">]>&xxe;"; - XmlException e = assertThrows(XmlException.class, () -> PropertyLists.parse(xxe)); + XMLParseException e = assertThrows(XMLParseException.class, () -> PropertyLists.parse(xxe)); assertTrue(e.getMessage().contains("undefined entity"), e.getMessage()); } @@ -103,43 +107,63 @@ void deepNestingIsRefused() { for (int i = 0; i < 10000; i++) { b.append(""); } - XmlException e = assertThrows(XmlException.class, () -> PropertyLists.parse(b.toString())); + XMLParseException e = assertThrows(XMLParseException.class, () -> PropertyLists.parse(b.toString())); assertTrue(e.getMessage().contains("nested deeper"), e.getMessage()); } @Test - void malformedInputIsAnXmlException() { - assertThrows(XmlException.class, () -> PropertyLists.parse("a")); - assertThrows(XmlException.class, () -> PropertyLists.parse("x")); - assertThrows(XmlException.class, () -> PropertyLists.parse("a")); - assertThrows(XmlException.class, () -> PropertyLists.parse("bplist00xyz".getBytes(StandardCharsets.UTF_8))); + void malformedInputIsAnXmlParseException() { + assertThrows(XMLParseException.class, () -> PropertyLists.parse("a")); + assertThrows(XMLParseException.class, () -> PropertyLists.parse("x")); + assertThrows(XMLParseException.class, () -> PropertyLists.parse("a")); + assertThrows(XMLParseException.class, () -> PropertyLists.parse("bplist00xyz".getBytes(StandardCharsets.UTF_8))); } @Test - void pullParserEvents() throws Exception { - XmlPullParser p = XmlPullParser.of("t"); - assertEquals(XmlPullParser.START_ELEMENT, p.next()); - assertEquals("a", p.getName()); - assertEquals("1", p.getAttributeValue("x")); - assertEquals("<", p.getAttributeValue("y")); - assertEquals(XmlPullParser.START_ELEMENT, p.next()); - assertEquals(XmlPullParser.END_ELEMENT, p.next()); - assertEquals("b", p.getName()); - assertEquals(XmlPullParser.TEXT, p.next()); - assertEquals("t", p.getText()); - assertEquals(XmlPullParser.END_ELEMENT, p.next()); - assertEquals(XmlPullParser.END_DOCUMENT, p.next()); - XmlElement root = XmlElement.parse("v"); + void sharedParserEventsAndElementHelpers() throws Exception { + final StringBuilder events = new StringBuilder(); + XMLParser p = new XMLParser() { + @Override + protected boolean startTag(String tag) { + events.append('<').append(tag); + return true; + } + + @Override + protected void attribute(String tag, String name, String value) { + events.append(' ').append(name).append('=').append(value); + } + + @Override + protected void endTag(String tag) { + events.append("'); + } + + @Override + protected void textElement(String text) { + events.append('[').append(text).append(']'); + } + }; + p.setStrict(true); + p.eventParser(new StringReader("t")); + assertEquals("[t]", events.toString()); + + XMLParser dom = new XMLParser(); + dom.setStrict(true); + Element root = dom.parse(new StringReader(" v ")); assertEquals(2, root.children("i").size()); - assertEquals("v", root.childText("i").isEmpty() ? root.children("i").get(1).getText() : "?"); + assertEquals("", root.childText("i")); + assertNull(root.childText("missing")); + assertNull(root.child("missing")); assertEquals("1", root.child("i").getAttribute("n")); + assertEquals("2", root.children("i").get(1).getAttribute("n")); } @Test void impossibleDatesAreRefusedAndYearZeroFloors() throws Exception { - assertThrows(XmlException.class, () -> PropertyLists.parse( + assertThrows(XMLParseException.class, () -> PropertyLists.parse( "2023-02-29T00:00:00Z")); - assertThrows(XmlException.class, () -> PropertyLists.parse( + assertThrows(XMLParseException.class, () -> PropertyLists.parse( "2024-04-31T00:00:00Z")); assertEquals(951782400000L, ((Date) PropertyLists.parse( "2000-02-29T00:00:00Z")).getTime()); diff --git a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java index dcb9698ecd4..2540d57d295 100644 --- a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java @@ -29,6 +29,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /// Strict mode keeps OOXML exactly as written; the default mode is unchanged. @@ -70,9 +71,9 @@ void strictIsCaseExact() { @Test void entitiesAreDecodedInTextAndAttributes() { - Element r = parse("<a> & b&c", true, false); + Element r = parse("<a> & b😀", true, false); assertEquals("x&yA", r.getAttribute("a")); - assertEquals(" & b&c", r.getChildAt(0).getText()); + assertEquals(" & b\ud83d\ude00", r.getChildAt(0).getText()); } @Test @@ -121,4 +122,102 @@ void strictTagSearchesAreExact() { assertEquals("good", r.getFirstChildByTagName("name").getChildAt(0).getText()); assertNull(r.getFirstChildByTagName("NAME")); } + + private static String failure(String xml) { + return assertThrows(XMLParseException.class, () -> parse(xml, true, false)).getMessage(); + } + + @Test + void malformedDocumentsAreRefused() { + assertTrue(failure("").contains("does not match")); + assertTrue(failure("").contains("end of document")); + assertTrue(failure("").contains("no open element")); + assertTrue(failure("").contains("second root")); + assertTrue(failure("junk").contains("outside the root")); + assertTrue(failure("junk").contains("outside the root")); + assertTrue(failure("").contains("not quoted")); + assertTrue(failure("").contains("duplicate attribute")); + assertTrue(failure("").contains("< in the value")); + assertTrue(failure("", "comment"); - continue; - } - if (in.startsWith("", "processing instruction"); - continue; - } - if (in.startsWith(""); - } - while (true) { - skipWhitespace(); - if (pos >= in.length()) { - throw error("unterminated start tag <" + tag + ">"); - } - char c = in.charAt(pos); - if (c == '>') { - pos++; - break; - } - if (c == '/') { - if (pos + 1 >= in.length() || in.charAt(pos + 1) != '>') { - throw error("expected /> in <" + tag + ">"); - } - pos += 2; - pendingEnd = true; - break; - } - String attr = readName(); - skipWhitespace(); - expect('='); - skipWhitespace(); - if (pos >= in.length()) { - throw error("unterminated attribute " + attr); - } - char quote = in.charAt(pos); - if (quote != '"' && quote != '\'') { - throw error("attribute " + attr + " is not quoted"); - } - pos++; - StringBuilder value = new StringBuilder(); - while (true) { - if (pos >= in.length()) { - throw error("unterminated value of attribute " + attr); - } - char v = in.charAt(pos); - if (v == quote) { - pos++; - break; - } - if (v == '<') { - throw error("< in the value of attribute " + attr); - } - if (v == '&') { - readReference(value); - } else { - value.append(v); - pos++; - } - } - if (attrNames.contains(attr)) { - throw error("duplicate attribute " + attr); - } - attrNames.add(attr); - attrValues.add(value.toString()); - } - if (stack.size() >= maxDepth) { - throw error("elements nested deeper than " + maxDepth); - } - stack.add(tag); - rootSeen = true; - name = tag; - return event = START_ELEMENT; - } - - private int readEndTag() throws XmlException { - pos += 2; - String tag = readName(); - skipWhitespace(); - expect('>'); - if (stack.isEmpty()) { - throw error("end tag with no open element"); - } - String open = stack.remove(stack.size() - 1); - if (!open.equals(tag)) { - throw error("end tag does not match <" + open + ">"); - } - name = tag; - return event = END_ELEMENT; - } - - private void readReference(StringBuilder out) throws XmlException { - int semi = in.indexOf(';', pos); - if (semi < 0 || semi - pos > 12) { - throw error("unterminated entity reference"); - } - String ref = in.substring(pos + 1, semi); - pos = semi + 1; - if ("amp".equals(ref)) { - out.append('&'); - } else if ("lt".equals(ref)) { - out.append('<'); - } else if ("gt".equals(ref)) { - out.append('>'); - } else if ("quot".equals(ref)) { - out.append('"'); - } else if ("apos".equals(ref)) { - out.append('\''); - } else if (ref.length() > 1 && ref.charAt(0) == '#') { - int code = parseCharRef(ref); - if (code < 0 || code > 0x10ffff || code == 0 || (code >= 0xd800 && code <= 0xdfff)) { - throw error("invalid character reference &" + ref + ";"); - } - if (code >= 0x10000) { - int v = code - 0x10000; - out.append((char) (0xd800 + (v >> 10))); - out.append((char) (0xdc00 + (v & 0x3ff))); - } else { - out.append((char) code); - } - } else { - // Deliberately no DTD-declared entities: see the class comment. - throw error("undefined entity &" + ref + ";"); - } - } - - private static int parseCharRef(String ref) { - boolean hex = ref.length() > 2 && (ref.charAt(1) == 'x' || ref.charAt(1) == 'X'); - int start = hex ? 2 : 1; - if (start >= ref.length()) { - return -1; - } - int value = 0; - for (int i = start; i < ref.length(); i++) { - int digit = Character.digit(ref.charAt(i), hex ? 16 : 10); - if (digit < 0) { - return -1; - } - value = value * (hex ? 16 : 10) + digit; - if (value > 0x10ffff) { - return -1; - } - } - return value; - } - - /// Skips `` (or any other `' && brackets <= 0) { - return; - } - } - throw error("unterminated declaration"); - } - - private void skipPast(String terminator, String what) throws XmlException { - int end = in.indexOf(terminator, pos + 2); - if (end < 0) { - throw error("unterminated " + what); - } - pos = end + terminator.length(); - } - - private String readName() throws XmlException { - int start = pos; - int len = in.length(); - while (pos < len) { - char c = in.charAt(pos); - if (c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '>' || c == '/' - || c == '=' || c == '<' || c == '"' || c == '\'') { - break; - } - pos++; - } - if (pos == start) { - throw error("expected a name"); - } - return in.substring(start, pos); - } - - private void expect(char c) throws XmlException { - if (pos >= in.length() || in.charAt(pos) != c) { - throw error("expected '" + c + "'"); - } - pos++; - } - - private void skipWhitespace() { - int len = in.length(); - while (pos < len && isSpace(in.charAt(pos))) { - pos++; - } - } - - private static boolean isSpace(char c) { - return c == ' ' || c == '\t' || c == '\r' || c == '\n'; - } - - private static boolean isWhitespace(CharSequence s) { - for (int i = 0; i < s.length(); i++) { - if (!isSpace(s.charAt(i))) { - return false; - } - } - return true; - } - - private XmlException error(String message) { - return new XmlException(message + " at offset " + pos); - } -} diff --git a/vm/backend/src/com/codename1/backend/xml/package-info.java b/vm/backend/src/com/codename1/backend/xml/package-info.java index 5070443c9de..a6c55cc660e 100644 --- a/vm/backend/src/com/codename1/backend/xml/package-info.java +++ b/vm/backend/src/com/codename1/backend/xml/package-info.java @@ -20,16 +20,14 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/// Reading XML on the server: a streaming pull parser, a small element tree and -/// an Apple property list reader. +/// Reading Apple property lists on the server. /// -/// [XmlPullParser] is the parser everything else is built on; [XmlElement] is a -/// tree for documents small enough to hold in memory; [PropertyLists] reads XML -/// property lists, including the one inside a signed provisioning profile. -/// -/// All three are safe for untrusted input by construction. A document type -/// declaration is skipped and never interpreted, so no external entity is ever -/// resolved and no entity is ever expanded beyond the five XML predefines and -/// numeric character references; nesting depth is capped. There is no switch -/// that turns either protection off. +/// [PropertyLists] reads XML property lists, including the one inside a signed +/// provisioning profile. XML itself is parsed by the shared +/// `com.codename1.xml.XMLParser`, the same parser the client uses, in strict +/// mode: a document type declaration is skipped and never interpreted, so no +/// external entity is ever resolved and no entity is ever expanded beyond the +/// five XML predefines and numeric character references; nesting depth is +/// capped; and a document that is not well formed is a +/// `com.codename1.xml.XMLParseException`. package com.codename1.backend.xml; From 45d6c30b1478d50edc520e8491f554eb84c9bff2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:23:51 +0100 Subject: [PATCH 20/25] ORM: a PostgreSQL default collation is not a declared one; exact PostgreSQL names; ICU collation names Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/impl/orm/SessionImpl.java | 23 ++++++--- .../processors/OrmAnnotationProcessor.java | 8 ++-- .../OrmAnnotationProcessorTest.java | 4 +- .../impl/orm/DeclaredCollationTest.java | 48 +++++++++++++++++++ 4 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 maven/javase/src/test/java/com/codename1/impl/orm/DeclaredCollationTest.java diff --git a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java index fa9d0d5a669..27fc1403ff5 100644 --- a/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java +++ b/CodenameOne/src/com/codename1/impl/orm/SessionImpl.java @@ -1176,17 +1176,26 @@ private String collateClause(Attribute attr) { if (attr.collation == null) { return ""; } - return " COLLATE " + ("postgresql".equals(sql.dialect()) ? q(attr.collation) : attr.collation); + // A hyphenated name (PostgreSQL's ICU collations, "und-x-icu") is + // only valid quoted; every engine takes a quoted collation name. + return " COLLATE " + ("postgresql".equals(sql.dialect()) || attr.collation.indexOf('-') >= 0 + ? q(attr.collation) : attr.collation); } /// Whether a found collation is the declared one. The PostgreSQL describe /// prefixes a nondeterministic one, which is part of what was declared. - /// SQLite reports a collation for key columns only; an unknown one passes. - static boolean declaredCollation(String declared, String found) { + /// PostgreSQL reports no collation for a column that uses its type's + /// default, which is the collation named "default" and matches only that; + /// its names are quoted identifiers, so case matters. SQLite reports a + /// collation for key columns only, so there an unknown one passes. + static boolean declaredCollation(String declared, String found, boolean postgresql) { if (found == null) { - return true; + return !postgresql || "default".equals(declared); } String name = found.startsWith("nondeterministic ") ? found.substring(17) : found; - return declared.equalsIgnoreCase(name) || declared.equalsIgnoreCase(found); + if (postgresql) { + return declared.equals(name); + } + return declared.equalsIgnoreCase(name); } /// The native column of a @Temporal field. DATETIME rather than TIMESTAMP on /// MySQL: its TIMESTAMP ends in 2038 and, unless explicit_defaults_for_timestamp @@ -1419,7 +1428,7 @@ public SchemaReport schemaReport() { // read back, so adopting such a schema keeps its semantics. boolean distinct = attr.id || uniqueIndexed(model, attr); String role = attr.id ? "Identifier " : "Unique column "; - if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5])) { + if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5], "postgresql".equals(sql.dialect()))) { errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, model.table(), attr.column, name + " uses collation " + found[5] + ", not the declared " + attr.collation)); } @@ -1542,7 +1551,7 @@ private void validateCollectionColumns(String table, List expected, L // SQLite lets a composite primary key column hold NULL, so only its // NOT NULL flag says the column is required there. boolean required = ((Number) found[2]).intValue() != 0 || primary && !"sqlite".equals(sql.dialect()); - if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5])) { + if (attr.collation != null && found.length > 5 && !declaredCollation(attr.collation, (String) found[5], "postgresql".equals(sql.dialect()))) { errors.add(new SchemaReport.Issue(SchemaReport.Kind.UNSUPPORTED, table, attr.column, "Collection column " + name + " uses collation " + found[5] + ", not the declared " + attr.collation)); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java index 55da1b199b7..250bb658b9b 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/OrmAnnotationProcessor.java @@ -572,7 +572,7 @@ private void processClass(AnnotatedClass cls, ProcessorContext ctx, boolean from if (collation != null && collation.length() > 0) { if (!collationName(collation)) { // Spliced into DDL as it is. - ctx.error(cls, "@Column(collation) must be letters, digits and underscores: " + ctx.error(cls, "@Column(collation) must be letters, digits, underscores and hyphens: " + pf.fieldName + " declares \"" + collation + "\""); } pf.collation = collation; @@ -1181,11 +1181,13 @@ && lowerOrDigit(name.charAt(i-1)) && lowerOrDigit(name.charAt(i+1))) return out.toString().toLowerCase(java.util.Locale.ROOT); } - /// Whether a `@Column(collation)` name is safe to splice into DDL. + /// Whether a `@Column(collation)` name is safe to splice into DDL. A + /// hyphen is allowed for ICU names (`und-x-icu`); such a name is emitted + /// quoted. static boolean collationName(String name) { for (int i = 0; i < name.length(); i++) { char c = name.charAt(i); - if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_')) { + if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '-')) { return false; } } diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java index 3fe8d95f88e..0993420eda9 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/OrmAnnotationProcessorTest.java @@ -167,7 +167,9 @@ public void collationMustBeAName() throws Exception { ProcessorContext ctx=runProcessor(classes,Collections.emptyList()); assertTrue(ctx.getErrors().toString(),ctx.getErrors().toString().contains("@Column(collation) must be letters")); assertTrue(OrmAnnotationProcessor.collationName("utf8mb4_0900_ai_ci")); - assertFalse(OrmAnnotationProcessor.collationName("en-x-icu")); + assertTrue(OrmAnnotationProcessor.collationName("en-x-icu")); + assertFalse(OrmAnnotationProcessor.collationName("en x")); + assertFalse(OrmAnnotationProcessor.collationName("a\"b")); } /// `@Column(insertable = false, updatable = false)` is a column only the diff --git a/maven/javase/src/test/java/com/codename1/impl/orm/DeclaredCollationTest.java b/maven/javase/src/test/java/com/codename1/impl/orm/DeclaredCollationTest.java new file mode 100644 index 00000000000..88e8e25e3c3 --- /dev/null +++ b/maven/javase/src/test/java/com/codename1/impl/orm/DeclaredCollationTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.orm; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class DeclaredCollationTest { + @Test + void postgresqlNullIsTheDefaultCollation() { + assertFalse(SessionImpl.declaredCollation("C", null, true)); + assertTrue(SessionImpl.declaredCollation("default", null, true)); + } + + @Test + void postgresqlComparesExactlyAndReadsTheNondeterministicMark() { + assertTrue(SessionImpl.declaredCollation("und-x-icu", "nondeterministic und-x-icu", true)); + assertFalse(SessionImpl.declaredCollation("C", "c", true)); + } + + @Test + void otherEnginesFoldCaseAndPassAnUnknownCollation() { + assertTrue(SessionImpl.declaredCollation("utf8mb4_0900_ai_ci", "UTF8MB4_0900_AI_CI", false)); + assertFalse(SessionImpl.declaredCollation("utf8mb4_bin", "utf8mb4_0900_ai_ci", false)); + assertTrue(SessionImpl.declaredCollation("NOCASE", null, false)); + } +} From 7a9931e5248020c3fa88191544b57b50a6eccfc6 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:23:23 +0100 Subject: [PATCH 21/25] Backend: one CivilTime helper for the UTC civil-date math Timestamps, PropertyLists, Credentials, Clock, Http1Date and Asn1 each had their own days-from-civil / civil-from-days / leap-year / pad copy. They now share com.codename1.impl.backend.CivilTime; each keeps its own parsing, separators and error behaviour. Asn1.parseTime now rejects a day past the month's end, as the other parsers already did. Co-Authored-By: Claude Opus 5.5 --- .../codename1/impl/backend/CivilTimeTest.java | 113 ++++++++++++++ .../src/com/codename1/backend/Http1Date.java | 59 +------- .../src/com/codename1/backend/aws/Clock.java | 52 +------ .../codename1/backend/aws/Credentials.java | 18 +-- .../com/codename1/backend/log/Timestamps.java | 24 +-- .../codename1/backend/security/pki/Asn1.java | 50 +----- .../codename1/backend/xml/PropertyLists.java | 16 +- .../com/codename1/impl/backend/CivilTime.java | 143 ++++++++++++++++++ 8 files changed, 287 insertions(+), 188 deletions(-) create mode 100644 maven/backend/src/test/java/com/codename1/impl/backend/CivilTimeTest.java create mode 100644 vm/backend/src/com/codename1/impl/backend/CivilTime.java diff --git a/maven/backend/src/test/java/com/codename1/impl/backend/CivilTimeTest.java b/maven/backend/src/test/java/com/codename1/impl/backend/CivilTimeTest.java new file mode 100644 index 00000000000..92b347b3f2f --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/impl/backend/CivilTimeTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.backend; + +import java.time.LocalDate; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class CivilTimeTest { + + private static int[] civil(long days) { + int[] out = new int[3]; + CivilTime.civilFromDays(days, out); + return out; + } + + private static String date(long days) { + return CivilTime.appendDate(new StringBuilder(), days, '-').toString(); + } + + @Test + void epochZero() { + assertEquals(0, CivilTime.daysFromCivil(1970, 1, 1)); + assertArrayEquals(new int[] {1970, 1, 1}, civil(0)); + assertEquals("1970-01-01", date(0)); + assertArrayEquals(new int[] {1969, 12, 31}, civil(-1)); + } + + @Test + void marchFirst1970() { + assertEquals(59, CivilTime.daysFromCivil(1970, 3, 1)); + assertArrayEquals(new int[] {1970, 3, 1}, civil(59)); + } + + @Test + void leapCenturies() { + assertTrue(CivilTime.isLeap(2000)); + assertFalse(CivilTime.isLeap(1900)); + assertTrue(CivilTime.isLeap(-4)); + assertTrue(CivilTime.isLeap(0)); + assertEquals(29, CivilTime.daysInMonth(2000, 2)); + assertEquals(28, CivilTime.daysInMonth(1900, 2)); + assertEquals(30, CivilTime.daysInMonth(1900, 4)); + long feb29 = CivilTime.daysFromCivil(2000, 2, 29); + assertEquals(LocalDate.of(2000, 2, 29).toEpochDay(), feb29); + assertArrayEquals(new int[] {2000, 3, 1}, civil(feb29 + 1)); + long feb28 = CivilTime.daysFromCivil(1900, 2, 28); + assertEquals(feb28 + 1, CivilTime.daysFromCivil(1900, 3, 1)); + assertArrayEquals(new int[] {1900, 3, 1}, civil(feb28 + 1)); + } + + @Test + void negativeYears() { + long d = CivilTime.daysFromCivil(-44, 3, 15); + assertEquals(LocalDate.of(-44, 3, 15).toEpochDay(), d); + assertArrayEquals(new int[] {-44, 3, 15}, civil(d)); + assertEquals("-0044-03-15", date(d)); + long y0 = CivilTime.daysFromCivil(0, 2, 29); + assertEquals(LocalDate.of(0, 2, 29).toEpochDay(), y0); + assertArrayEquals(new int[] {-1, 12, 31}, civil(CivilTime.daysFromCivil(0, 1, 1) - 1)); + } + + @Test + void roundTripAgainstJavaTime() { + int[] out = new int[3]; + for (long days = -800000; days <= 800000; days += 7) { + CivilTime.civilFromDays(days, out); + LocalDate ref = LocalDate.ofEpochDay(days); + assertEquals(ref.getYear(), out[0]); + assertEquals(ref.getMonthValue(), out[1]); + assertEquals(ref.getDayOfMonth(), out[2]); + assertEquals(days, CivilTime.daysFromCivil(out[0], out[1], out[2])); + assertEquals(ref.lengthOfMonth(), CivilTime.daysInMonth(out[0], out[1])); + } + } + + @Test + void padding() { + assertEquals("007", CivilTime.pad(new StringBuilder(), 7, 3).toString()); + assertEquals("12345", CivilTime.pad(new StringBuilder(), 12345, 2).toString()); + assertEquals("0", CivilTime.pad(new StringBuilder(), 0, 1).toString()); + assertEquals("00", CivilTime.pad(new StringBuilder(), 0, 2).toString()); + assertEquals("-05", CivilTime.pad(new StringBuilder(), -5, 2).toString()); + assertEquals("-9223372036854775808", + CivilTime.pad(new StringBuilder(), Long.MIN_VALUE, 4).toString()); + assertEquals("9223372036854775807", + CivilTime.pad(new StringBuilder(), Long.MAX_VALUE, 4).toString()); + } +} diff --git a/vm/backend/src/com/codename1/backend/Http1Date.java b/vm/backend/src/com/codename1/backend/Http1Date.java index 6ceb8cd11e3..f61751eb84c 100644 --- a/vm/backend/src/com/codename1/backend/Http1Date.java +++ b/vm/backend/src/com/codename1/backend/Http1Date.java @@ -22,6 +22,8 @@ */ package com.codename1.backend; +import com.codename1.impl.backend.CivilTime; + /// The one date format HTTP/1.1 requires on the wire ("Sun, 06 Nov 1994 08:49:37 /// GMT"), formatted and parsed from epoch milliseconds directly. /// @@ -37,14 +39,8 @@ public final class Http1Date { private Http1Date() { } - // JavaAPI's Math has no floorDiv/floorMod, and integer division in Java - // truncates toward zero -- which for a pre-1970 timestamp gives the wrong day. private static long floorDiv(long x, long y) { - long q = x / y; - if ((x % y != 0) && ((x < 0) != (y < 0))) { - q--; - } - return q; + return CivilTime.floorDiv(x, y); } private static long floorMod(long x, long y) { @@ -57,7 +53,8 @@ public static String format(long millis) { int secondOfDay = (int) floorMod(seconds, 86400L); // 1970-01-01 was a Thursday, which is why DAYS starts there. int dayOfWeek = (int) floorMod(days, 7L); - int[] civil = civilFromDays(days); + int[] civil = new int[3]; + CivilTime.civilFromDays(days, civil); StringBuilder out = new StringBuilder(29); out.append(DAYS[dayOfWeek]).append(", "); two(out, civil[2]).append(' ').append(MONTHS[civil[1] - 1]).append(' '); @@ -124,11 +121,11 @@ public static long parse(String value) { // one written. A second of 60 is allowed because a leap second is // spelled that way. if (day < 1 || year < 1 || hour < 0 || minute < 0 || second < 0 - || day > daysInMonth(year, month) || hour > 23 || minute > 59 + || day > CivilTime.daysInMonth(year, month) || hour > 23 || minute > 59 || second > 60) { return -1; } - long days = daysFromCivil(year, month, day); + long days = CivilTime.daysFromCivil(year, month, day); // The weekday, which the shape test above never looked at: the first // three characters were accepted as-is, so "Xxx, 06 Nov 9999 ..." read // as a year-9999 date and StaticFiles.isNotModified() answered 304 to @@ -168,52 +165,10 @@ private static int digits(String v, int from, int len) { return out; } - /// Days in a month, so a date that does not exist is not silently moved. - private static int daysInMonth(int year, int month) { - switch (month) { - case 2: - boolean leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; - return leap ? 29 : 28; - case 4: case 6: case 9: case 11: - return 30; - default: - return 31; - } - } - private static StringBuilder two(StringBuilder out, int value) { if (value < 10) { out.append('0'); } return out.append(value); } - - /* - * Howard Hinnant's civil-date algorithms: exact for every date in range, with - * no leap-year special cases to get wrong. The shift moves the epoch to - * 0000-03-01 so February -- the only month whose length varies -- lands at the - * end of the year and drops out of the arithmetic. - */ - private static int[] civilFromDays(long days) { - long z = days + 719468L; - long era = floorDiv(z, 146097L); - long doe = z - era * 146097L; - long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - long y = yoe + era * 400L; - long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - long mp = (5 * doy + 2) / 153; - long d = doy - (153 * mp + 2) / 5 + 1; - long m = mp < 10 ? mp + 3 : mp - 9; - return new int[]{(int) (m <= 2 ? y + 1 : y), (int) m, (int) d}; - } - - private static long daysFromCivil(int year, int month, int day) { - long y = year - (month <= 2 ? 1 : 0); - long era = floorDiv(y, 400L); - long yoe = y - era * 400L; - long mp = month > 2 ? month - 3 : month + 9; - long doy = (153 * mp + 2) / 5 + day - 1; - long doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - return era * 146097L + doe - 719468L; - } } diff --git a/vm/backend/src/com/codename1/backend/aws/Clock.java b/vm/backend/src/com/codename1/backend/aws/Clock.java index 0efbf8cdba7..3530196733d 100644 --- a/vm/backend/src/com/codename1/backend/aws/Clock.java +++ b/vm/backend/src/com/codename1/backend/aws/Clock.java @@ -22,6 +22,8 @@ */ package com.codename1.backend.aws; +import com.codename1.impl.backend.CivilTime; + /// The ISO basic timestamp AWS signs with: yyyyMMdd'T'HHmmss'Z', always UTC. /// /// Written out by hand rather than through SimpleDateFormat because the translated @@ -29,9 +31,6 @@ /// Arabic-Indic digit or a locale that renders the year differently produces a /// signature the service cannot reproduce, and the only symptom is a 403. final class Clock { - private static final int[] DAYS_IN_MONTH = - {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - private Clock() { } @@ -44,36 +43,11 @@ static String timestamp(long millis) { if (millis < 0 && (millis % 1000L) != 0) { seconds--; // floor, so a pre-epoch instant does not round toward zero } - long days = floorDiv(seconds, 86400L); + long days = CivilTime.floorDiv(seconds, 86400L); int secondOfDay = (int) (seconds - days * 86400L); - int year = 1970; - while (true) { - int length = isLeap(year) ? 366 : 365; - if (days >= length) { - days -= length; - year++; - } else if (days < 0) { - year--; - days += isLeap(year) ? 366 : 365; - } else { - break; - } - } - int month = 0; - while (true) { - int length = DAYS_IN_MONTH[month] + (month == 1 && isLeap(year) ? 1 : 0); - if (days < length) { - break; - } - days -= length; - month++; - } - StringBuilder out = new StringBuilder(16); - pad(out, year, 4); - pad(out, month + 1, 2); - pad(out, (int) days + 1, 2); + CivilTime.appendDate(out, days, (char) 0); out.append('T'); pad(out, secondOfDay / 3600, 2); pad(out, (secondOfDay / 60) % 60, 2); @@ -82,23 +56,7 @@ static String timestamp(long millis) { return out.toString(); } - private static long floorDiv(long value, long divisor) { - long q = value / divisor; - if ((value % divisor != 0) && ((value < 0) != (divisor < 0))) { - q--; - } - return q; - } - - private static boolean isLeap(int year) { - return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; - } - private static void pad(StringBuilder out, int value, int width) { - String text = String.valueOf(value); - for (int iter = text.length() ; iter < width ; iter++) { - out.append('0'); - } - out.append(text); + CivilTime.pad(out, value, width); } } diff --git a/vm/backend/src/com/codename1/backend/aws/Credentials.java b/vm/backend/src/com/codename1/backend/aws/Credentials.java index 4c981973865..3a9a642429e 100644 --- a/vm/backend/src/com/codename1/backend/aws/Credentials.java +++ b/vm/backend/src/com/codename1/backend/aws/Credentials.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.aws; +import com.codename1.impl.backend.CivilTime; import java.io.IOException; import java.util.Map; @@ -411,7 +412,6 @@ static long expiryMillis(String iso) { int hour = Integer.parseInt(iso.substring(11, 13)); int minute = Integer.parseInt(iso.substring(14, 16)); int second = Integer.parseInt(iso.substring(17, 19)); - int[] lengths = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; // A DATE THAT EXISTS. Digits alone are not enough: the arithmetic below // is a running total, so February the 31st simply carries into March // and an hour of 99 adds four days. Both are LATER than the provider @@ -422,28 +422,16 @@ static long expiryMillis(String iso) { || hour > 23 || minute > 59 || second > 60) { return 0; } - int maxDay = lengths[month - 1] + (month == 2 && isLeap(year) ? 1 : 0); - if (day > maxDay) { + if (day > CivilTime.daysInMonth(year, month)) { return 0; } - long days = 0; - for (int y = 1970 ; y < year ; y++) { - days += isLeap(y) ? 366 : 365; - } - for (int m = 0 ; m < month - 1 ; m++) { - days += lengths[m] + (m == 1 && isLeap(year) ? 1 : 0); - } - days += day - 1; + long days = CivilTime.daysFromCivil(year, month, day); return ((days * 24L + hour) * 60L + minute) * 60L * 1000L + second * 1000L; } catch (NumberFormatException err) { return 0; } } - private static boolean isLeap(int year) { - return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; - } - /// Whether a field the credential endpoint returned is missing in any of the /// ways that leave it unusable. /// diff --git a/vm/backend/src/com/codename1/backend/log/Timestamps.java b/vm/backend/src/com/codename1/backend/log/Timestamps.java index b14177c4188..0ae4613059c 100644 --- a/vm/backend/src/com/codename1/backend/log/Timestamps.java +++ b/vm/backend/src/com/codename1/backend/log/Timestamps.java @@ -22,6 +22,8 @@ */ package com.codename1.backend.log; +import com.codename1.impl.backend.CivilTime; + /// ISO-8601 UTC timestamps without java.time or a formatter. final class Timestamps { private Timestamps() { @@ -29,21 +31,9 @@ private Timestamps() { /// Appends `millis` as `yyyy-MM-ddTHH:mm:ss.SSSZ`. static void appendIso(StringBuilder out, long millis) { - long days = Math.floorDiv(millis, 86400000L); + long days = CivilTime.floorDiv(millis, 86400000L); long ofDay = millis - days * 86400000L; - // Civil date from days since the epoch (H. Hinnant's days_from_civil inverse). - long z = days + 719468; - long era = Math.floorDiv(z, 146097); - long doe = z - era * 146097; - long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - long mp = (5 * doy + 2) / 153; - long day = doy - (153 * mp + 2) / 5 + 1; - long month = mp < 10 ? mp + 3 : mp - 9; - long year = yoe + era * 400 + (month <= 2 ? 1 : 0); - pad(out, year, 4).append('-'); - pad(out, month, 2).append('-'); - pad(out, day, 2).append('T'); + CivilTime.appendDate(out, days, '-').append('T'); pad(out, ofDay / 3600000L, 2).append(':'); pad(out, ofDay / 60000L % 60, 2).append(':'); pad(out, ofDay / 1000L % 60, 2).append('.'); @@ -51,10 +41,6 @@ static void appendIso(StringBuilder out, long millis) { } private static StringBuilder pad(StringBuilder out, long value, int width) { - String digits = Long.toString(value); - for (int i = digits.length(); i < width; i++) { - out.append('0'); - } - return out.append(digits); + return CivilTime.pad(out, value, width); } } diff --git a/vm/backend/src/com/codename1/backend/security/pki/Asn1.java b/vm/backend/src/com/codename1/backend/security/pki/Asn1.java index 3b28e71e1b9..7c23d6a2100 100644 --- a/vm/backend/src/com/codename1/backend/security/pki/Asn1.java +++ b/vm/backend/src/com/codename1/backend/security/pki/Asn1.java @@ -22,6 +22,8 @@ */ package com.codename1.backend.security.pki; +import com.codename1.impl.backend.CivilTime; + import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -556,11 +558,11 @@ static long parseTime(String text, boolean utc) throws IOException { int hour = digits(text, yearDigits + 4, 2); int minute = digits(text, yearDigits + 6, 2); int second = digits(text, yearDigits + 8, 2); - if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 - || second > 59) { + if (month < 1 || month > 12 || day < 1 || day > CivilTime.daysInMonth(year, month) + || hour > 23 || minute > 59 || second > 59) { throw new IOException("Not an RFC 5280 time: " + text); } - long days = daysFromCivil(year, month, day); + long days = CivilTime.daysFromCivil(year, month, day); return ((days * 24 + hour) * 60 + minute) * 60000L + second * 1000L; } @@ -570,18 +572,10 @@ static String formatTime(long millis) { if (millis < 0 && seconds * 1000L != millis) { seconds--; } - long days = seconds / 86400L; - if (seconds < 0 && days * 86400L != seconds) { - days--; - } + long days = CivilTime.floorDiv(seconds, 86400L); int secondOfDay = (int) (seconds - days * 86400L); - int[] civil = civilFromDays(days); StringBuilder out = new StringBuilder(); - pad(out, civil[0], 4); - out.append('-'); - pad(out, civil[1], 2); - out.append('-'); - pad(out, civil[2], 2); + CivilTime.appendDate(out, days, '-'); out.append('T'); pad(out, secondOfDay / 3600, 2); out.append(':'); @@ -592,11 +586,7 @@ static String formatTime(long millis) { } private static void pad(StringBuilder out, int value, int width) { - String text = Integer.toString(value); - for (int iter = text.length() ; iter < width ; iter++) { - out.append('0'); - } - out.append(text); + CivilTime.pad(out, value, width); } private static int digits(String text, int from, int count) throws IOException { @@ -611,30 +601,6 @@ private static int digits(String text, int from, int count) throws IOException { return out; } - /// Days since 1970-01-01 of a proleptic Gregorian date (Howard Hinnant's - /// algorithm). - static long daysFromCivil(int year, int month, int day) { - int y = month <= 2 ? year - 1 : year; - int era = (y >= 0 ? y : y - 399) / 400; - int yoe = y - era * 400; - int doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - return era * 146097L + doe - 719468L; - } - - static int[] civilFromDays(long days) { - long z = days + 719468L; - long era = (z >= 0 ? z : z - 146096) / 146097; - int doe = (int) (z - era * 146097); - int yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - int doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - int mp = (5 * doy + 2) / 153; - int day = doy - (153 * mp + 2) / 5 + 1; - int month = mp < 10 ? mp + 3 : mp - 9; - long year = yoe + era * 400 + (month <= 2 ? 1 : 0); - return new int[] {(int) year, month, day}; - } - static String hex(byte[] value) { char[] digits = "0123456789abcdef".toCharArray(); StringBuilder out = new StringBuilder(value.length * 2); diff --git a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java index 73a57428fc3..56bc8fa09f1 100644 --- a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java +++ b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java @@ -23,6 +23,7 @@ package com.codename1.backend.xml; import com.codename1.backend.Base64; +import com.codename1.impl.backend.CivilTime; import com.codename1.xml.Element; import com.codename1.xml.XMLParseException; import com.codename1.xml.XMLParser; @@ -279,21 +280,10 @@ private static Date parseDate(String s) { || hour < 0 || hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 60) { throw new XMLParseException("bad " + s); } - boolean leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); - int length = month == 2 ? (leap ? 29 : 28) - : month == 4 || month == 6 || month == 9 || month == 11 ? 30 : 31; - if (day > length) { + if (day > CivilTime.daysInMonth(year, month)) { throw new XMLParseException("bad " + s); } - // Days from the civil calendar (H. Hinnant's algorithm), proleptic Gregorian. - // y is -1 for January and February of year 0, so the era floors. - int y = month <= 2 ? year - 1 : year; - int era = (y >= 0 ? y : y - 399) / 400; - int yoe = y - era * 400; - int mp = (month + 9) % 12; - int doy = (153 * mp + 2) / 5 + day - 1; - int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - long days = (long) era * 146097 + doe - 719468; + long days = CivilTime.daysFromCivil(year, month, day); long seconds = days * 86400L + hour * 3600L + minute * 60L + second; return new Date(seconds * 1000L); } diff --git a/vm/backend/src/com/codename1/impl/backend/CivilTime.java b/vm/backend/src/com/codename1/impl/backend/CivilTime.java new file mode 100644 index 00000000000..23d8e08b906 --- /dev/null +++ b/vm/backend/src/com/codename1/impl/backend/CivilTime.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.backend; + +/// UTC civil-date arithmetic on the proleptic Gregorian calendar, shared by the +/// backend's hand-written date formats: log timestamps, HTTP dates, plist +/// dates, AWS timestamps and X.509 times. +/// +/// Hand-written rather than java.time on purpose: the translated runtime's date +/// parsing is locale-aware, and the log path must not allocate. Nothing here +/// parses or validates a format; each caller keeps its own separators and +/// error behaviour and takes only the arithmetic from this class. +public final class CivilTime { + private CivilTime() { + } + + /// Floor division. JavaAPI's Math has no floorDiv, and Java's `/` truncates + /// toward zero, which gives the wrong day before 1970. + public static long floorDiv(long x, long y) { + long q = x / y; + if (x % y != 0 && (x < 0) != (y < 0)) { + q--; + } + return q; + } + + /// Whether `y` is a Gregorian leap year. + public static boolean isLeap(long y) { + return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0); + } + + /// The length of month `m` (1 to 12) of year `y`. + public static int daysInMonth(long y, int m) { + switch (m) { + case 2: + return isLeap(y) ? 29 : 28; + case 4: + case 6: + case 9: + case 11: + return 30; + default: + return 31; + } + } + + /// Days since 1970-01-01 of the date `y-m-d` (Howard Hinnant's + /// days_from_civil). It does not validate: day 31 of February is + /// normalised into March, so a caller checks `daysInMonth` first. + public static long daysFromCivil(long y, int m, int d) { + long yy = m <= 2 ? y - 1 : y; + long era = floorDiv(yy, 400L); + long yoe = yy - era * 400L; + long mp = m > 2 ? m - 3 : m + 9; + long doy = (153 * mp + 2) / 5 + d - 1; + long doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097L + doe - 719468L; + } + + /// Writes the civil date of `days` since 1970-01-01 into `out` as year, + /// month (1 to 12) and day (1 to 31), so a caller can reuse one array. + public static void civilFromDays(long days, int[] out) { + long z = days + 719468L; + long era = floorDiv(z, 146097L); + long doe = z - era * 146097L; + long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + long mp = (5 * doy + 2) / 153; + long m = mp < 10 ? mp + 3 : mp - 9; + out[0] = (int) (yoe + era * 400L + (m <= 2 ? 1 : 0)); + out[1] = (int) m; + out[2] = (int) (doy - (153 * mp + 2) / 5 + 1); + } + + /// Appends the date of `days` since 1970-01-01 as year (four digits at + /// least), month and day (two digits each), with `separator` between the + /// fields or nothing when it is 0. Allocates nothing. + public static StringBuilder appendDate(StringBuilder out, long days, char separator) { + long z = days + 719468L; + long era = floorDiv(z, 146097L); + long doe = z - era * 146097L; + long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + long mp = (5 * doy + 2) / 153; + long m = mp < 10 ? mp + 3 : mp - 9; + pad(out, yoe + era * 400L + (m <= 2 ? 1 : 0), 4); + if (separator != 0) { + out.append(separator); + } + pad(out, m, 2); + if (separator != 0) { + out.append(separator); + } + return pad(out, doy - (153 * mp + 2) / 5 + 1, 2); + } + + /// Appends `value` in at least `width` digits, zero-filled on the left, + /// without allocating. A negative value is a `-` followed by its padded + /// magnitude (`-0044`), never zeros in front of the sign. + public static StringBuilder pad(StringBuilder out, long value, int width) { + if (value == Long.MIN_VALUE) { + return out.append("-9223372036854775808"); + } + long v = value; + if (v < 0) { + out.append('-'); + v = -v; + } + int len = 1; + long div = 1; + while (v / div >= 10) { + div *= 10; + len++; + } + for (int i = len; i < width; i++) { + out.append('0'); + } + for (; div > 0; div /= 10) { + out.append((char) ('0' + (v / div) % 10)); + } + return out; + } +} From 6f8ecf84684c348f032300ad674eef714a057f23 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:28:06 +0100 Subject: [PATCH 22/25] Backend: JsonCodec and CronSchedule use CivilTime Co-Authored-By: Claude Opus 5.5 --- .../codename1/impl/backend/CronSchedule.java | 39 +++------------ .../com/codename1/impl/backend/JsonCodec.java | 48 ++----------------- 2 files changed, 11 insertions(+), 76 deletions(-) diff --git a/vm/backend/src/com/codename1/impl/backend/CronSchedule.java b/vm/backend/src/com/codename1/impl/backend/CronSchedule.java index 8029ed634d3..a808ee3f969 100644 --- a/vm/backend/src/com/codename1/impl/backend/CronSchedule.java +++ b/vm/backend/src/com/codename1/impl/backend/CronSchedule.java @@ -524,11 +524,7 @@ private long toUtc(long local) { } static long floorDiv(long value, long divisor) { - long q = value / divisor; - if ((value % divisor != 0) && ((value < 0) != (divisor < 0))) { - q--; - } - return q; + return CivilTime.floorDiv(value, divisor); } static long floorMod(long value, long divisor) { @@ -536,40 +532,17 @@ static long floorMod(long value, long divisor) { } static int daysInMonth(int year, int month) { - switch (month) { - case 2: - boolean leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; - return leap ? 29 : 28; - case 4: - case 6: - case 9: - case 11: - return 30; - default: - return 31; - } + return CivilTime.daysInMonth(year, month); } static long daysFromCivil(int y, int m, int d) { - int adjusted = y - (m <= 2 ? 1 : 0); - long era = (adjusted >= 0 ? adjusted : adjusted - 399) / 400; - int yoe = (int) (adjusted - era * 400); - int doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; - int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - return era * 146097L + doe - 719468L; + return CivilTime.daysFromCivil(y, m, d); } static int[] civilFromDays(long z) { - long shifted = z + 719468L; - long era = (shifted >= 0 ? shifted : shifted - 146096) / 146097; - long doe = shifted - era * 146097; - long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - long y = yoe + era * 400; - long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - long mp = (5 * doy + 2) / 153; - long d = doy - (153 * mp + 2) / 5 + 1; - long m = mp + (mp < 10 ? 3 : -9); - return new int[] {(int) (y + (m <= 2 ? 1 : 0)), (int) m, (int) d}; + int[] out = new int[3]; + CivilTime.civilFromDays(z, out); + return out; } @Override diff --git a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java index 169e861f39b..0fcd90fa815 100644 --- a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java +++ b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java @@ -252,30 +252,11 @@ public static void writeIsoDate(Date value, ByteSink out) { return; } long millis = value.getTime(); - // Floored by hand: vm/JavaAPI's Math has no floorDiv. - long days = millis / 86400000L; - if (millis % 86400000L < 0) { - days--; - } + long days = CivilTime.floorDiv(millis, 86400000L); long ms = millis - days * 86400000L; - // civil_from_days, Howard Hinnant's algorithm; the inverse of daysFromCivil. - long z = days + 719468; - long era = (z >= 0 ? z : z - 146096) / 146097; - long doe = z - era * 146097; - long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; - long y = yoe + era * 400; - long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); - long mp = (5 * doy + 2) / 153; - long d = doy - (153 * mp + 2) / 5 + 1; - long m = mp < 10 ? mp + 3 : mp - 9; - if (m <= 2) { - y++; - } StringBuilder sb = new StringBuilder(24); sb.append('"'); - pad(sb, y, 4).append('-'); - pad(sb, m, 2).append('-'); - pad(sb, d, 2).append('T'); + CivilTime.appendDate(sb, days, '-').append('T'); pad(sb, ms / 3600000L, 2).append(':'); pad(sb, ms / 60000L % 60, 2).append(':'); pad(sb, ms / 1000L % 60, 2).append('.'); @@ -284,15 +265,7 @@ public static void writeIsoDate(Date value, ByteSink out) { } private static StringBuilder pad(StringBuilder sb, long v, int width) { - if (v < 0) { - sb.append('-'); - v = -v; - } - String digits = Long.toString(v); - for (int i = digits.length(); i < width; i++) { - sb.append('0'); - } - return sb.append(digits); + return CivilTime.pad(sb, v, width); } private static String describe(Object got) { @@ -527,21 +500,10 @@ private static int digits(String s, int from, int count) { } private static int daysIn(int year, int month) { - if (month == 2) { - boolean leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; - return leap ? 29 : 28; - } - return month == 4 || month == 6 || month == 9 || month == 11 ? 30 : 31; + return CivilTime.daysInMonth(year, month); } - /// Days from 1970-01-01 to the given proleptic Gregorian date; Howard - /// Hinnant's algorithm, which needs no calendar class. private static long daysFromCivil(int year, int month, int day) { - int y = month <= 2 ? year - 1 : year; - int era = (y >= 0 ? y : y - 399) / 400; - int yoe = y - era * 400; - int doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day - 1; - int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - return era * 146097L + doe - 719468; + return CivilTime.daysFromCivil(year, month, day); } } From 6b38fb767881d06fbc3b30703d0f6cd953306791 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:40:31 +0100 Subject: [PATCH 23/25] XMLParser: restore setStrict semantics, put hardening behind setRejectMalformed setStrict(true) again only reports to the callback, as on master, so Mappers.fromXml and callbacks that continue keep working. The new setRejectMalformed(true) implies strict and adds throwing on malformed input, entity refusal, the depth cap and DOCTYPE skipping. It also refuses C0 control characters and &#X, skips comments and PIs in a DOCTYPE internal subset whole, reports line and column, and throws on I/O errors with the IOException as cause. Outside it, an I/O error with no callback is printed rather than lost. PropertyLists uses the new mode. Co-Authored-By: Claude Opus 5.5 --- .../com/codename1/xml/XMLParseException.java | 46 +++- .../src/com/codename1/xml/XMLParser.java | 233 +++++++++++++----- .../xml/XMLParserStrictModeTest.java | 83 ++++++- .../codename1/backend/xml/PropertyLists.java | 4 +- 4 files changed, 292 insertions(+), 74 deletions(-) diff --git a/CodenameOne/src/com/codename1/xml/XMLParseException.java b/CodenameOne/src/com/codename1/xml/XMLParseException.java index 960e1110e67..5c2fd3c1b28 100644 --- a/CodenameOne/src/com/codename1/xml/XMLParseException.java +++ b/CodenameOne/src/com/codename1/xml/XMLParseException.java @@ -22,7 +22,8 @@ */ package com.codename1.xml; -/// Thrown by an [XMLParser] in strict mode (see [XMLParser#setStrict(boolean)]) +/// Thrown by an [XMLParser] refusing malformed input (see +/// [XMLParser#setRejectMalformed(boolean)]) /// when the document is not well formed, nests deeper than /// [XMLParser#setMaxDepth(int)], or references an entity the parser does not /// define. @@ -32,12 +33,51 @@ /// keeps working. @com.codename1.impl.SharedWithBackend public class XMLParseException extends IllegalArgumentException { - /// Creates the exception with the description of what is wrong. + private final int line; + private final int column; + + /// Creates an exception with no position. /// /// #### Parameters /// - /// - `message`: the description + /// - `message`: what is wrong public XMLParseException(String message) { super(message); + line = -1; + column = -1; + } + + /// Creates an exception at a position, which the message names as + /// "line 3, column 7: ...". + /// + /// #### Parameters + /// + /// - `message`: what is wrong + /// + /// - `line`: the line of the offending character, from 1 + /// + /// - `column`: its column, from 1 (0 before the first character of a line) + public XMLParseException(String message, int line, int column) { + super("line " + line + ", column " + column + ": " + message); + this.line = line; + this.column = column; + } + + /// The line of the error, from 1, or -1 when it has no position. + /// + /// #### Returns + /// + /// the line + public int getLine() { + return line; + } + + /// The column of the error, from 1, or -1 when it has no position. + /// + /// #### Returns + /// + /// the column + public int getColumn() { + return column; } } diff --git a/CodenameOne/src/com/codename1/xml/XMLParser.java b/CodenameOne/src/com/codename1/xml/XMLParser.java index ef76447e5fe..0fb67a5d239 100644 --- a/CodenameOne/src/com/codename1/xml/XMLParser.java +++ b/CodenameOne/src/com/codename1/xml/XMLParser.java @@ -73,6 +73,10 @@ public class XMLParser { boolean includeWhitespacesBetweenTags; // For HTML white spaces between tags are significant to seperate words, in XML less so and it mostly creates garbage elements (Text with one space) private boolean eventParser; private boolean strict; + private boolean rejectMalformed; + // The position of the last character read, for error messages. + private int line = 1; + private int column; private boolean stripNamespacePrefixes; // How many parseTagContent calls are active. 1 is document level: the // synthetic ROOT of parse(), or no element at all for eventParser. @@ -147,14 +151,35 @@ private int read(Reader is) throws IOException { } c = buffer[buffOffset]; buffOffset++; - + advance((char) c); return c; } + /// Moves the error position past `c`, refusing a control character the + /// XML character range excludes when malformed input is rejected. + private void advance(char c) { + if (c == '\n') { + line++; + column = 0; + } else { + column++; + if (rejectMalformed && c < 0x20 && c != '\t' && c != '\r') { + throw malformed("illegal control character 0x" + Integer.toHexString(c)); + } + } + } + + /// An [XMLParseException] at the current position. + private XMLParseException malformed(String message) { + return new XMLParseException(message, line, column); + } + private void resetBuffer() { buffOffset = 0; buffSize = -1; contentDepth = 0; + line = 1; + column = 0; openCount = 0; rootSeen = false; if (pool != null) { @@ -467,9 +492,18 @@ public Element parse(Reader is) { try { parseTagContent(rootElement, is); } catch (IOException ioe) { - // Reported through the callback rather than a log: the parser is - // shared with the server runtime, which has no Codename One Log. - notifyError(ParserCallback.ERROR_IO, null, null, null, "I/O error while parsing: " + ioe.getMessage()); + // Not Log.e: the parser is shared with the server runtime, which + // has no Codename One Log. + if (rejectMalformed) { + XMLParseException e = malformed("I/O error while parsing: " + ioe.getMessage()); + e.initCause(ioe); + throw e; + } + if (parserCallback == null) { + ioe.printStackTrace(); //NOPMD AvoidPrintStackTrace - core cannot use Log here, see above + } else { + notifyError(ParserCallback.ERROR_IO, null, null, null, "I/O error while parsing: " + ioe.getMessage()); + } } if (rootElement.getNumChildren() == 0) { notifyError(ParserCallback.ERROR_NO_ROOTS, null, null, null, "XML document contains no root element."); @@ -552,8 +586,8 @@ public void eventParser(Reader r) throws IOException { resetBuffer(); eventParser = true; parseTagContent(null, r); - if (strict && !rootSeen) { - throw new XMLParseException("no root element"); + if (rejectMalformed && !rootSeen) { + throw malformed("no root element"); } } @@ -653,7 +687,7 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException while (val != -1) { if (c == '<') { if ((includeWhitespacesBetweenTags) && (leadingSpace) && (!inText) && (element != null) && (element.getNumChildren() > 0) - && !(strict && contentDepth == 1)) { + && !(rejectMalformed && contentDepth == 1)) { leadingSpace = false; text.clear(); text.add(' '); @@ -681,7 +715,7 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException Element childElement = parseTag(is); - if (childElement == END_TAG && strict) { + if (childElement == END_TAG && rejectMalformed) { strictEndTag(is); return; } @@ -744,7 +778,7 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException inEntity = false; } } else if (c == '&') { //start char entity - if (strict) { + if (rejectMalformed) { readStrictReference(is, text); } else { charEntity.clear(); @@ -766,16 +800,19 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException } end++; } + for (int k = start; k < end; k++) { + advance(b[k]); + } if (end > start) { text.add(b, start, end - start); buffOffset = end; } } - } else if (strict && contentDepth == 1 && (ws(c) || c == (char) 0xfeff)) { + } else if (rejectMalformed && contentDepth == 1 && (ws(c) || c == (char) 0xfeff)) { // whitespace, or a byte order mark, around the root element leadingSpace = true; - } else if (strict && contentDepth == 1) { - throw new XMLParseException("text outside the root element"); + } else if (rejectMalformed && contentDepth == 1) { + throw malformed("text outside the root element"); } else if (keepAllText || !isWhiteSpace(c)) { text.clear(); inText = true; @@ -785,7 +822,7 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException } if (c == '&') { //text starts with a character entity (i.e.  ) // The & is not included in the string we accumulate - if (strict) { + if (rejectMalformed) { readStrictReference(is, text); } else { charEntity.clear(); @@ -801,8 +838,8 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException val = read(is); c = (char) val; } - if (strict && contentDepth > 1) { - throw new XMLParseException("unexpected end of document inside <" + openName() + ">"); + if (rejectMalformed && contentDepth > 1) { + throw malformed("unexpected end of document inside <" + openName() + ">"); } } @@ -818,7 +855,7 @@ private void strictEndTag(Reader is) throws IOException { int i = read(is); while (i != '>') { if (i < 0) { - throw new XMLParseException("unterminated end tag "); + throw malformed("unterminated end tag "); } closingTag.add((char) i); i = read(is); @@ -828,11 +865,11 @@ private void strictEndTag(Reader is) throws IOException { } String ct = name(closingTag); if (contentDepth < 2 || openCount == 0) { - throw new XMLParseException("end tag with no open element"); + throw malformed("end tag with no open element"); } String open = openNames[openCount - 1]; if (!ct.equals(open)) { - throw new XMLParseException("end tag does not match <" + open + ">"); + throw malformed("end tag does not match <" + open + ">"); } if (eventParser) { endTag(ct); @@ -853,14 +890,21 @@ private void readStrictReference(Reader is, CharBuf out) throws IOException { break; } if (i < 0 || ref.len >= 32 || !isLegalStrictReferenceCharacter((char) i)) { - throw new XMLParseException("unterminated entity reference &" + ref); + throw malformed("unterminated entity reference &" + ref); } ref.add((char) i); } String name = ref.toString(); + if (name.length() > 1 && name.charAt(0) == '#' && name.charAt(1) == 'X') { + throw malformed("character reference &" + name + "; must use a lowercase x"); + } String decoded = CharEntities.decodeXml(name, userDefinedCharEntities); if (decoded == null) { - throw new XMLParseException("undefined entity &" + name + ";"); + throw malformed("undefined entity &" + name + ";"); + } + char d = decoded.charAt(0); + if (decoded.length() == 1 && d < 0x20 && d != '\t' && d != '\n' && d != '\r') { + throw malformed("character reference &" + name + "; is an illegal control character"); } out.add(decoded); } @@ -875,16 +919,35 @@ private static boolean isLegalStrictReferenceCharacter(char c) { /// lenient parser always has. private char tagChar(Reader is) throws IOException { int i = read(is); - if (i < 0 && strict) { - throw new XMLParseException("unexpected end of document inside a tag"); + if (i < 0 && rejectMalformed) { + throw malformed("unexpected end of document inside a tag"); } return (char) i; } + /// Reads up to and including `terminator`, refusing the end of the document. + private void skipPast(Reader is, String terminator) throws IOException { + int len = terminator.length(); + char[] last = new char[len]; + int seen = 0; + while (true) { + int i = read(is); + if (i < 0) { + throw malformed("unterminated declaration"); + } + System.arraycopy(last, 1, last, 0, len - 1); + last[len - 1] = (char) i; + seen++; + if (seen >= len && terminator.equals(new String(last))) { + return; + } + } + } + /// What `parseTag` answers when the document ends inside a tag. private Element eofInTag() { - if (strict) { - throw new XMLParseException("unexpected end of document inside a tag"); + if (rejectMalformed) { + throw malformed("unexpected end of document inside a tag"); } return END_TAG; } @@ -892,7 +955,7 @@ private Element eofInTag() { /// Parses the content of the element `tn` just opened, keeping strict /// mode's stack of open names and its depth cap. private void tagContent(Element element, Reader is, String tn) throws IOException { - if (!strict) { + if (!rejectMalformed) { parseTagContent(element, is); return; } @@ -918,12 +981,16 @@ private Element skipDeclaration(Reader is, char first, char second) throws IOExc int brackets = 0; char quote = 0; int pending = 2; + int pushed = -1; while (true) { int i; if (pending == 2) { i = first; } else if (pending == 1) { i = second; + } else if (pushed >= 0) { + i = pushed; + pushed = -1; } else { i = read(is); } @@ -931,7 +998,7 @@ private Element skipDeclaration(Reader is, char first, char second) throws IOExc pending--; } if (i < 0 || i == 0xffff) { - throw new XMLParseException("unterminated declaration"); + throw malformed("unterminated declaration"); } char ch = (char) i; if (quote != 0) { @@ -940,6 +1007,25 @@ private Element skipDeclaration(Reader is, char first, char second) throws IOExc } } else if (ch == '"' || ch == '\'') { quote = ch; + } else if (ch == '<' && brackets > 0) { + // A comment or processing instruction in the internal subset + // is skipped whole: a quote or bracket inside it means nothing. + int n = read(is); + if (n == '?') { + skipPast(is, "?>"); + } else if (n == '!') { + int m = read(is); + if (m == '-') { + if (read(is) != '-') { + throw malformed("malformed comment in a declaration"); + } + skipPast(is, "-->"); + } else { + pushed = m; + } + } else { + pushed = n; + } } else if (ch == '[') { brackets++; } else if (ch == ']') { @@ -1002,7 +1088,7 @@ protected Element parseTag(Reader is) throws IOException { char c2 = tagChar(is); if ((c == '-') && (c2 == '-')) { //comment return parseCommentOrXMLDeclaration(is, "-->"); - } else if (strict && !((c == '[') && (c2 == 'C'))) { + } else if (rejectMalformed && !((c == '[') && (c2 == 'C'))) { return skipDeclaration(is, c, c2); } else if ((c == '[') && (c2 == 'C')) { // CDATA? c = tagChar(is); @@ -1015,13 +1101,13 @@ protected Element parseTag(Reader is) throws IOException { } } if (idx == clen) { //found CDATA - if (strict && contentDepth == 1) { - throw new XMLParseException("text outside the root element"); + if (rejectMalformed && contentDepth == 1) { + throw malformed("text outside the root element"); } return parseCommentOrXMLDeclaration(is, "]]>"); //parse doctypes i.e. as comments as well - i.e. ignore them } else { // some other unknown tag - if (strict) { - throw new XMLParseException("malformed CDATA section"); + if (rejectMalformed) { + throw malformed("malformed CDATA section"); } return parseCommentOrXMLDeclaration(is, ">"); //parse doctypes i.e. as comments as well - i.e. ignore them } @@ -1060,18 +1146,18 @@ protected Element parseTag(Reader is) throws IOException { boolean processTag = true; String tn = name(tagName); - if (strict) { + if (rejectMalformed) { if (tagName.len == 0) { - throw new XMLParseException("expected a tag name after <"); + throw malformed("expected a tag name after <"); } if (contentDepth == 1) { if (rootSeen) { - throw new XMLParseException("a second root element <" + tn + ">"); + throw malformed("a second root element <" + tn + ">"); } rootSeen = true; } if (openCount >= maxDepth) { - throw new XMLParseException("elements nested deeper than " + maxDepth); + throw malformed("elements nested deeper than " + maxDepth); } tagAttributes.clear(); } @@ -1195,8 +1281,8 @@ protected Element parseTag(Reader is) throws IOException { if ((c == '"') || (c == '\'')) { quote = c; - } else if (strict) { - throw new XMLParseException("attribute " + curAttribute + " of <" + tn + "> is not quoted"); + } else if (rejectMalformed) { + throw malformed("attribute " + curAttribute + " of <" + tn + "> is not quoted"); } else { curValue.add(c); } @@ -1219,9 +1305,9 @@ protected Element parseTag(Reader is) throws IOException { c = (char) i; } else if ((quote == ' ') && ((c == '/') || (c == '>') || (ws(c)))) { ended = true; - } else if (strict && c == '<') { - throw new XMLParseException("< in the value of attribute " + curAttribute + " of <" + tn + ">"); - } else if (strict && c == '&') { + } else if (rejectMalformed && c == '<') { + throw malformed("< in the value of attribute " + curAttribute + " of <" + tn + ">"); + } else if (rejectMalformed && c == '&') { readStrictReference(is, curValue); } else if (c == '&') { if (inEntity) { @@ -1256,13 +1342,13 @@ protected Element parseTag(Reader is) throws IOException { String an = attributeName(curAttribute); String av = value(curValue); - if (strict) { + if (rejectMalformed) { if (tagAttributes.contains(an)) { - throw new XMLParseException("duplicate attribute " + an + " in <" + tn + ">"); + throw malformed("duplicate attribute " + an + " in <" + tn + ">"); } tagAttributes.add(an); if (!ws(c) && c != '>' && c != '/') { - throw new XMLParseException("expected whitespace after attribute " + an + " of <" + tn + ">"); + throw malformed("expected whitespace after attribute " + an + " of <" + tn + ">"); } } if (eventParser) { @@ -1338,8 +1424,8 @@ protected Element parseCommentOrXMLDeclaration(Reader is, String endTag) throws while (!ended) { int in = read(is); if (in == -1) { - if (strict) { - throw new XMLParseException("unterminated " + ("-->".equals(endTag) ? "comment" + if (rejectMalformed) { + throw malformed("unterminated " + ("-->".equals(endTag) ? "comment" : "]]>".equals(endTag) ? "CDATA section" : "declaration")); } // input stream ended abruptly @@ -1427,8 +1513,8 @@ protected void notifyError(int errorId, String tag, String attribute, String val throw new IllegalArgumentException(description); } } - if (strict) { - throw new XMLParseException(description); + if (rejectMalformed) { + throw malformed(description); } } @@ -1502,15 +1588,27 @@ public void setCaseSensitive(boolean caseSensitive) { /// ` ` keeps its space; /// /// - whitespace is the four XML whitespace characters and `isWhiteSpace` - /// is not consulted; + /// is not consulted. + /// + /// #### Parameters + /// + /// - `strict`: true for strict XML parsing + public void setStrict(boolean strict) { + this.strict = strict; + } + + /// Turns on refusing malformed input, for documents that may be hostile + /// (off by default). It implies `setStrict(true)` and adds to it: /// /// - a document that is not well formed is refused with an - /// [XMLParseException]: a mismatched, stray or missing end tag, a second - /// root element or text outside the root, an unquoted or duplicate - /// attribute, a `<` in an attribute value, an unterminated comment, CDATA - /// section or declaration, and an undefined or malformed entity - /// reference. Every error reported to a [ParserCallback] is thrown, even - /// when the callback asks to continue; + /// [XMLParseException] naming the line and column: a mismatched, stray + /// or missing end tag, a second root element or text outside the root, + /// an unquoted or duplicate attribute, a `<` in an attribute value, an + /// unterminated comment, CDATA section or declaration, an undefined or + /// malformed entity reference, a control character other than tab, line + /// feed and carriage return, and an I/O error (the exception's cause). + /// Every error reported to a [ParserCallback] is thrown, even when the + /// callback asks to continue; /// /// - only the five XML entities, character references and names added with /// [#addCharEntity(String, int)] are defined. A document type @@ -1524,9 +1622,22 @@ public void setCaseSensitive(boolean caseSensitive) { /// /// #### Parameters /// - /// - `strict`: true for strict XML parsing - public void setStrict(boolean strict) { - this.strict = strict; + /// - `rejectMalformed`: true to refuse malformed input + public void setRejectMalformed(boolean rejectMalformed) { + this.rejectMalformed = rejectMalformed; + if (rejectMalformed) { + strict = true; + } + } + + /// Returns true if malformed input is refused, see + /// `setRejectMalformed(boolean)`. + /// + /// #### Returns + /// + /// true when malformed input is refused + public boolean isRejectMalformed() { + return rejectMalformed; } /// Returns true if strict XML parsing is on, see `setStrict(boolean)`. @@ -1538,9 +1649,9 @@ public boolean isStrict() { return strict; } - /// Sets the deepest element nesting strict mode accepts; a deeper document - /// is an [XMLParseException]. The default is 256. The lenient mode does - /// not check nesting. + /// Sets the deepest element nesting `setRejectMalformed(boolean)` accepts; a deeper document + /// is an [XMLParseException]. The default is 256. Otherwise nesting is + /// not checked. /// /// #### Parameters /// @@ -1552,7 +1663,7 @@ public void setMaxDepth(int maxDepth) { this.maxDepth = maxDepth; } - /// Returns the deepest element nesting strict mode accepts, see + /// Returns the deepest element nesting `setRejectMalformed(boolean)` accepts, see /// `setMaxDepth(int)`. /// /// #### Returns diff --git a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java index 2540d57d295..4cb94ced3d3 100644 --- a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java @@ -71,9 +71,9 @@ void strictIsCaseExact() { @Test void entitiesAreDecodedInTextAndAttributes() { - Element r = parse("<a> & b😀", true, false); + Element r = parse("<a> & b&c", true, false); assertEquals("x&yA", r.getAttribute("a")); - assertEquals(" & b\ud83d\ude00", r.getChildAt(0).getText()); + assertEquals(" & b&c", r.getChildAt(0).getText()); } @Test @@ -124,7 +124,74 @@ void strictTagSearchesAreExact() { } private static String failure(String xml) { - return assertThrows(XMLParseException.class, () -> parse(xml, true, false)).getMessage(); + return assertThrows(XMLParseException.class, () -> rejecting().parse(new StringReader(xml))).getMessage(); + } + + private static XMLParser rejecting() { + XMLParser p = new XMLParser(); + p.setRejectMalformed(true); + return p; + } + + @Test + void rejectingDecodesSupplementaryReferences() { + Element r = rejecting().parse(new StringReader("b😀")); + assertEquals("b\ud83d\ude00", r.getChildAt(0).getText()); + assertTrue(rejecting().isStrict()); + } + + @Test + void strictAloneStaysTolerant() { + // Mappers.fromXml parses this way: what master accepted still parses. + XMLParser p = new XMLParser(); + p.setStrict(true); + Element r = p.parse(new StringReader("x y")); + assertEquals("a", r.getTagName()); + assertTrue(r.getChildAt(0).getText().startsWith("x")); + assertEquals("a", p.parse(new StringReader("")).getTagName()); + XMLParser q = new XMLParser(); + q.setStrict(true); + q.setParserCallback((id, tag, attribute, value, description) -> true); + assertEquals("a", q.parse(new StringReader("")).getTagName()); + } + + @Test + void errorsNameTheLineAndColumn() { + XMLParseException e = assertThrows(XMLParseException.class, + () -> rejecting().parse(new StringReader("\n\n "))); + assertEquals(3, e.getLine()); + assertTrue(e.getMessage().startsWith("line 3, column "), e.getMessage()); + assertTrue(e.getColumn() > 0); + } + + @Test + void controlCharactersAreRefused() { + assertTrue(failure("x\u0001").contains("control character")); + assertTrue(failure("").contains("control character")); + assertTrue(failure("").contains("control character")); + assertTrue(failure("A").contains("lowercase x")); + assertEquals("\t\nA", rejecting().parse(new StringReader(" A")).getChildAt(0).getText()); + } + + @Test + void ioErrorsKeepTheirCause() { + java.io.IOException boom = new java.io.IOException("boom"); + java.io.Reader r = new java.io.Reader() { + public int read(char[] b, int o, int l) throws java.io.IOException { + throw boom; + } + + public void close() { + } + }; + XMLParseException e = assertThrows(XMLParseException.class, () -> rejecting().parse(r)); + assertSame(boom, e.getCause()); + } + + @Test + void doctypeCommentsAndInstructionsAreSkippedWhole() { + Element r = rejecting().parse(new StringReader(" -->?>]>")); + assertEquals("r", r.getTagName()); } @Test @@ -160,7 +227,7 @@ void badEntitiesAreRefused() { @Test void anEntityTheApplicationAddsIsDefined() { XMLParser p = new XMLParser(); - p.setStrict(true); + p.setRejectMalformed(true); p.addCharEntity("nbsp", 160); assertEquals("\u00a0", p.parse(new StringReader(" ")).getChildAt(0).getText()); } @@ -172,7 +239,7 @@ void doctypeEntitiesAreNeverExpanded() { assertTrue(failure(xxe).contains("undefined entity &xxe;")); String bomb = "]>&b;"; assertTrue(failure(bomb).contains("undefined entity &b;")); - Element r = parse("]>\nok", true, false); + Element r = rejecting().parse(new StringReader("]>\nok")); assertEquals("ok", r.getChildAt(0).getText()); } @@ -184,7 +251,7 @@ void nestingIsCapped() { } assertTrue(failure(b.toString()).contains("nested deeper than 256")); XMLParser p = new XMLParser(); - p.setStrict(true); + p.setRejectMalformed(true); p.setMaxDepth(2); assertEquals(2, p.getMaxDepth()); assertEquals("b", p.parse(new StringReader("")).getChildAt(0).getTagName()); @@ -196,14 +263,14 @@ void nestingIsCapped() { @Test void aByteOrderMarkAndTheDeclarationAreSkipped() { - Element r = parse("\ufeff\n\n", true, false); + Element r = rejecting().parse(new StringReader("\ufeff\n\n")); assertEquals("r", r.getTagName()); } @Test void eventModeIsStrictToo() { XMLParser p = new XMLParser(); - p.setStrict(true); + p.setRejectMalformed(true); assertThrows(XMLParseException.class, () -> p.eventParser(new StringReader(""))); assertThrows(XMLParseException.class, () -> p.eventParser(new StringReader(" "))); } diff --git a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java index 56bc8fa09f1..f3651b4a5c8 100644 --- a/vm/backend/src/com/codename1/backend/xml/PropertyLists.java +++ b/vm/backend/src/com/codename1/backend/xml/PropertyLists.java @@ -68,7 +68,7 @@ /// signature is not verified. Binary property lists (`bplist00`) are not /// supported and are reported as such. /// -/// Parsing goes through the shared [XMLParser] in strict mode, so a document +/// Parsing goes through the shared [XMLParser] refusing malformed input, so a document /// type declaration is never interpreted and nesting is bounded: a hostile /// plist cannot fetch a URL, expand entities or exhaust the stack. public final class PropertyLists { @@ -155,7 +155,7 @@ public static byte[] extractXml(byte[] data) { /// default nesting cap. private static Element parseXml(Reader in) { XMLParser parser = new XMLParser(); - parser.setStrict(true); + parser.setRejectMalformed(true); Element root = parser.parse(in); if (root == null) { throw new XMLParseException("no root element"); From bc35111b364a81fe4c50cd4316b1914be1667fd9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:47:31 +0100 Subject: [PATCH 24/25] Backend: call CivilTime directly; test Asn1 rejects a day past month end Co-Authored-By: Claude Opus 5.5 --- .../backend/security/pki/PkiTest.java | 9 ++++++++ .../codename1/impl/backend/CronSchedule.java | 22 ++++-------------- .../com/codename1/impl/backend/JsonCodec.java | 23 +++++-------------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/maven/backend/src/test/java/com/codename1/backend/security/pki/PkiTest.java b/maven/backend/src/test/java/com/codename1/backend/security/pki/PkiTest.java index 48d6f3022ae..546d8cc3ae3 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/pki/PkiTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/pki/PkiTest.java @@ -340,4 +340,13 @@ void timesAreExact() throws Exception { assertThrows(IOException.class, () -> Asn1.parseTime("700101000000", true)); } + @Test + void timesRejectADayPastTheMonthEnd() throws Exception { + assertThrows(IOException.class, () -> Asn1.parseTime("240230120000Z", true)); + assertThrows(IOException.class, () -> Asn1.parseTime("19000229120000Z", false)); + assertThrows(IOException.class, () -> Asn1.parseTime("20250431000000Z", false)); + assertEquals(951825600000L, Asn1.parseTime("000229120000Z", true)); + assertEquals(951825600000L, Asn1.parseTime("20000229120000Z", false)); + } + } diff --git a/vm/backend/src/com/codename1/impl/backend/CronSchedule.java b/vm/backend/src/com/codename1/impl/backend/CronSchedule.java index a808ee3f969..edcdae93b2a 100644 --- a/vm/backend/src/com/codename1/impl/backend/CronSchedule.java +++ b/vm/backend/src/com/codename1/impl/backend/CronSchedule.java @@ -374,7 +374,7 @@ public long next(long afterMillis) { long t = afterMillis - floorMod(afterMillis, 1000L) + 1000L; for (int guard = 0 ; guard < SEARCH_DAYS ; guard++) { long local = t + offsetAt(t); - long day = floorDiv(local, DAY); + long day = CivilTime.floorDiv(local, DAY); int timeOfDay = (int) (local - day * DAY); int[] civil = civilFromDays(day); int month = civil[1]; @@ -425,7 +425,7 @@ private boolean dayMatches(int year, int month, int day, int dow) { return false; } if (lastDayOfMonth) { - return day == daysInMonth(year, month); + return day == CivilTime.daysInMonth(year, month); } return (daysOfMonth & (1L << day)) != 0; } @@ -473,7 +473,7 @@ private static long firstDayOfNextMonth(int[] civil) { m = 1; y++; } - return daysFromCivil(y, m, 1); + return CivilTime.daysFromCivil(y, m, 1); } /// Milliseconds east of UTC at the instant `utc`. @@ -483,7 +483,7 @@ private int offsetAt(long utc) { } int raw = timeZone.getRawOffset(); long standard = utc + raw; - long day = floorDiv(standard, DAY); + long day = CivilTime.floorDiv(standard, DAY); int[] civil = civilFromDays(day); int dow = (int) floorMod(day + 4, 7); return timeZone.getOffset(1, civil[0], civil[1] - 1, civil[2], dow + 1, @@ -523,20 +523,8 @@ private long toUtc(long local) { return local - offsetAt(utc); } - static long floorDiv(long value, long divisor) { - return CivilTime.floorDiv(value, divisor); - } - static long floorMod(long value, long divisor) { - return value - floorDiv(value, divisor) * divisor; - } - - static int daysInMonth(int year, int month) { - return CivilTime.daysInMonth(year, month); - } - - static long daysFromCivil(int y, int m, int d) { - return CivilTime.daysFromCivil(y, m, d); + return value - CivilTime.floorDiv(value, divisor) * divisor; } static int[] civilFromDays(long z) { diff --git a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java index 0fcd90fa815..20a68930b18 100644 --- a/vm/backend/src/com/codename1/impl/backend/JsonCodec.java +++ b/vm/backend/src/com/codename1/impl/backend/JsonCodec.java @@ -257,17 +257,13 @@ public static void writeIsoDate(Date value, ByteSink out) { StringBuilder sb = new StringBuilder(24); sb.append('"'); CivilTime.appendDate(sb, days, '-').append('T'); - pad(sb, ms / 3600000L, 2).append(':'); - pad(sb, ms / 60000L % 60, 2).append(':'); - pad(sb, ms / 1000L % 60, 2).append('.'); - pad(sb, ms % 1000L, 3).append("Z\""); + CivilTime.pad(sb, ms / 3600000L, 2).append(':'); + CivilTime.pad(sb, ms / 60000L % 60, 2).append(':'); + CivilTime.pad(sb, ms / 1000L % 60, 2).append('.'); + CivilTime.pad(sb, ms % 1000L, 3).append("Z\""); out.putAscii(sb.toString()); } - private static StringBuilder pad(StringBuilder sb, long v, int width) { - return CivilTime.pad(sb, v, width); - } - private static String describe(Object got) { if (got == null) { return "null"; @@ -416,7 +412,7 @@ static long parseIso8601(String s) { int year = digits(s, 0, 4); int month = digits(s, 5, 2); int day = digits(s, 8, 2); - if (year < 0 || month < 1 || month > 12 || day < 1 || day > daysIn(year, month)) { + if (year < 0 || month < 1 || month > 12 || day < 1 || day > CivilTime.daysInMonth(year, month)) { return Long.MIN_VALUE; } int hour = 0; @@ -478,7 +474,7 @@ static long parseIso8601(String s) { return Long.MIN_VALUE; } } - long days = daysFromCivil(year, month, day); + long days = CivilTime.daysFromCivil(year, month, day); return ((days * 24 + hour) * 60 + minute - offsetMinutes) * 60000L + second * 1000L + millis; } @@ -499,11 +495,4 @@ private static int digits(String s, int from, int count) { return v; } - private static int daysIn(int year, int month) { - return CivilTime.daysInMonth(year, month); - } - - private static long daysFromCivil(int year, int month, int day) { - return CivilTime.daysFromCivil(year, month, day); - } } From 93eb63cbb3e9cca7af8b14b4a976772e3fe6f19c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:17:55 +0100 Subject: [PATCH 25/25] XMLParser: position tracking only when rejecting, I/O callback cannot abort, effective strict flag Line/column tracking and the control-character check run only in reject mode, counted inside the existing text scan, so default and HTML parsing pay one boolean test. Outside reject mode an I/O error is reported to the callback but its answer is ignored, returning the partial tree as master did. setRejectMalformed no longer changes what isStrict reports, and turning it off restores the mode setStrict chose. Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/xml/XMLParser.java | 50 ++++++++++++------- .../xml/XMLParserStrictModeTest.java | 48 +++++++++++++++++- 2 files changed, 79 insertions(+), 19 deletions(-) diff --git a/CodenameOne/src/com/codename1/xml/XMLParser.java b/CodenameOne/src/com/codename1/xml/XMLParser.java index 0fb67a5d239..058dcad2752 100644 --- a/CodenameOne/src/com/codename1/xml/XMLParser.java +++ b/CodenameOne/src/com/codename1/xml/XMLParser.java @@ -72,7 +72,10 @@ public class XMLParser { ParserCallback parserCallback; boolean includeWhitespacesBetweenTags; // For HTML white spaces between tags are significant to seperate words, in XML less so and it mostly creates garbage elements (Text with one space) private boolean eventParser; + // The strict behaviour in effect: setStrict(true) or setRejectMalformed(true). private boolean strict; + // What setStrict set, which isStrict reports. + private boolean strictRequested; private boolean rejectMalformed; // The position of the last character read, for error messages. private int line = 1; @@ -151,19 +154,21 @@ private int read(Reader is) throws IOException { } c = buffer[buffOffset]; buffOffset++; - advance((char) c); + if (rejectMalformed) { + advance((char) c); + } return c; } - /// Moves the error position past `c`, refusing a control character the - /// XML character range excludes when malformed input is rejected. + /// Reject mode only: moves the error position past `c`, refusing a + /// control character the XML character range excludes. private void advance(char c) { if (c == '\n') { line++; column = 0; } else { column++; - if (rejectMalformed && c < 0x20 && c != '\t' && c != '\r') { + if (c < 0x20 && c != '\t' && c != '\r') { throw malformed("illegal control character 0x" + Integer.toHexString(c)); } } @@ -502,7 +507,9 @@ public Element parse(Reader is) { if (parserCallback == null) { ioe.printStackTrace(); //NOPMD AvoidPrintStackTrace - core cannot use Log here, see above } else { - notifyError(ParserCallback.ERROR_IO, null, null, null, "I/O error while parsing: " + ioe.getMessage()); + // Told, but never allowed to abort: master returned the partial + // tree here, and existing callbacks have never seen ERROR_IO. + parserCallback.parsingError(ParserCallback.ERROR_IO, null, null, null, "I/O error while parsing: " + ioe.getMessage()); } } if (rootElement.getNumChildren() == 0) { @@ -793,15 +800,23 @@ private void parseTagContentImpl(Element element, Reader is) throws IOException int start = buffOffset; int end = start; int lim = buffSize; - while (end < lim) { - char d = b[end]; - if (d == '<' || d == '&') { - break; + if (rejectMalformed) { + while (end < lim) { + char d = b[end]; + if (d == '<' || d == '&') { + break; + } + advance(d); + end++; + } + } else { + while (end < lim) { + char d = b[end]; + if (d == '<' || d == '&') { + break; + } + end++; } - end++; - } - for (int k = start; k < end; k++) { - advance(b[k]); } if (end > start) { text.add(b, start, end - start); @@ -1594,7 +1609,8 @@ public void setCaseSensitive(boolean caseSensitive) { /// /// - `strict`: true for strict XML parsing public void setStrict(boolean strict) { - this.strict = strict; + this.strictRequested = strict; + this.strict = strict || rejectMalformed; } /// Turns on refusing malformed input, for documents that may be hostile @@ -1625,9 +1641,7 @@ public void setStrict(boolean strict) { /// - `rejectMalformed`: true to refuse malformed input public void setRejectMalformed(boolean rejectMalformed) { this.rejectMalformed = rejectMalformed; - if (rejectMalformed) { - strict = true; - } + this.strict = strictRequested || rejectMalformed; } /// Returns true if malformed input is refused, see @@ -1646,7 +1660,7 @@ public boolean isRejectMalformed() { /// /// true for strict XML parsing public boolean isStrict() { - return strict; + return strictRequested; } /// Sets the deepest element nesting `setRejectMalformed(boolean)` accepts; a deeper document diff --git a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java index 4cb94ced3d3..fcd5c5a3e5c 100644 --- a/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/xml/XMLParserStrictModeTest.java @@ -137,7 +137,53 @@ private static XMLParser rejecting() { void rejectingDecodesSupplementaryReferences() { Element r = rejecting().parse(new StringReader("b😀")); assertEquals("b\ud83d\ude00", r.getChildAt(0).getText()); - assertTrue(rejecting().isStrict()); + assertTrue(rejecting().isRejectMalformed()); + } + + @Test + void turningRejectOffRestoresTheModeSetStrictChose() { + XMLParser p = rejecting(); + assertTrue(!p.isStrict()); + p.setRejectMalformed(false); + assertTrue(!p.isStrict()); + // lenient again: case folded, nothing thrown + assertEquals("doc", p.parse(new StringReader("b&c")).getTagName()); + XMLParser q = new XMLParser(); + q.setStrict(true); + q.setRejectMalformed(true); + q.setRejectMalformed(false); + assertTrue(q.isStrict()); + assertEquals("Doc", q.parse(new StringReader("")).getTagName()); + } + + @Test + void anIoErrorWithACallbackThatStopsStillReturnsThePartialTree() { + java.io.Reader r = new java.io.Reader() { + private boolean sent; + + public int read(char[] b, int o, int l) throws java.io.IOException { + if (sent) { + throw new java.io.IOException("boom"); + } + sent = true; + "".getChars(0, 11, b, o); + return 11; + } + + public void close() { + } + }; + final int[] seen = new int[1]; + XMLParser p = new XMLParser(); + p.setParserCallback((id, tag, attribute, value, description) -> { + if (id == ParserCallback.ERROR_IO) { + seen[0]++; + } + return false; + }); + Element e = p.parse(r); + assertEquals(1, seen[0]); + assertEquals("a", e.getTagName()); } @Test