From c5d764d36fb4fd48c7713dca7207f7a30ae8820d Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 07:52:51 +0100 Subject: [PATCH 01/11] Move and rewrite strlen methods --- .../foreign/AbstractMemorySegmentImpl.java | 4 + .../foreign/SegmentBulkOperations.java | 222 ++++++++++++++--- .../jdk/internal/foreign/StringSupport.java | 224 ++---------------- test/jdk/java/foreign/TestStringEncoding.java | 25 +- .../java/lang/foreign/InternalStrLen.java | 28 ++- 5 files changed, 256 insertions(+), 247 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/AbstractMemorySegmentImpl.java b/src/java.base/share/classes/jdk/internal/foreign/AbstractMemorySegmentImpl.java index 97417efaa8ca3..a52881b047b18 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/AbstractMemorySegmentImpl.java +++ b/src/java.base/share/classes/jdk/internal/foreign/AbstractMemorySegmentImpl.java @@ -889,23 +889,27 @@ public void setAtIndex(AddressLayout layout, long index, MemorySegment value) { layout.varHandle().set((MemorySegment)this, index * layout.byteSize(), value); } + @ForceInline @Override public String getString(long offset) { return getString(offset, sun.nio.cs.UTF_8.INSTANCE); } + @ForceInline @Override public String getString(long offset, Charset charset) { Objects.requireNonNull(charset); return StringSupport.read(this, offset, charset); } + @ForceInline @Override public void setString(long offset, String str) { Objects.requireNonNull(str); setString(offset, str, sun.nio.cs.UTF_8.INSTANCE); } + @ForceInline @Override public void setString(long offset, String str, Charset charset) { Objects.requireNonNull(charset); diff --git a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java index d928f8fc425fe..f9ba0c240442b 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java +++ b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java @@ -28,12 +28,13 @@ import jdk.internal.misc.ScopedMemoryAccess; import jdk.internal.util.Architecture; import jdk.internal.util.ArraysSupport; -import jdk.internal.util.ByteArrayLittleEndian; import jdk.internal.vm.annotation.ForceInline; import jdk.internal.vm.annotation.Stable; import java.lang.foreign.MemorySegment; +import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; + /** * This class contains optimized bulk operation methods that operate on one or several * memory segments. @@ -50,6 +51,7 @@ public final class SegmentBulkOperations { private SegmentBulkOperations() {} private static final ScopedMemoryAccess SCOPED_MEMORY_ACCESS = ScopedMemoryAccess.getScopedMemoryAccess(); + private static final long LONG_MASK = ~7L; // The last three bits are zero // All the threshold values below MUST be a power of two and should preferably be // greater or equal to 2^3. @@ -75,21 +77,21 @@ public static MemorySegment fill(AbstractMemorySegmentImpl dst, byte value) { int offset = 0; // 0...0X...X000 final int limit = (int) (dst.length & (NATIVE_THRESHOLD_FILL - 8)); - for (; offset < limit; offset += 8) { + for (; offset < limit; offset += Long.BYTES) { SCOPED_MEMORY_ACCESS.putLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, longValue, !Architecture.isLittleEndian()); } int remaining = (int) dst.length - limit; // 0...0X00 - if (remaining >= 4) { + if (remaining >= Integer.BYTES) { SCOPED_MEMORY_ACCESS.putIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, (int) longValue, !Architecture.isLittleEndian()); - offset += 4; - remaining -= 4; + offset += Integer.BYTES; + remaining -= Integer.BYTES; } // 0...00X0 - if (remaining >= 2) { + if (remaining >= Short.BYTES) { SCOPED_MEMORY_ACCESS.putShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + offset, (short) longValue, !Architecture.isLittleEndian()); - offset += 2; - remaining -= 2; + offset += Short.BYTES; + remaining -= Short.BYTES; } // 0...000X if (remaining == 1) { @@ -123,26 +125,26 @@ public static void copy(AbstractMemorySegmentImpl src, long srcOffset, // is an overlap, we could tolerate one particular direction of overlap (but not the other). // 0...0X...X000 - final int limit = (int) (size & (NATIVE_THRESHOLD_COPY - 8)); + final int limit = (int) (size & (NATIVE_THRESHOLD_COPY - Long.BYTES)); int offset = 0; - for (; offset < limit; offset += 8) { + for (; offset < limit; offset += Long.BYTES) { final long v = SCOPED_MEMORY_ACCESS.getLongUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian()); SCOPED_MEMORY_ACCESS.putLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian()); } int remaining = (int) size - offset; // 0...0X00 - if (remaining >= 4) { + if (remaining >= Integer.BYTES) { final int v = SCOPED_MEMORY_ACCESS.getIntUnaligned(src.sessionImpl(), src.unsafeGetBase(),src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian()); SCOPED_MEMORY_ACCESS.putIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian()); - offset += 4; - remaining -= 4; + offset += Integer.BYTES; + remaining -= Integer.BYTES; } // 0...00X0 - if (remaining >= 2) { + if (remaining >= Short.BYTES) { final short v = SCOPED_MEMORY_ACCESS.getShortUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcOffset + offset, !Architecture.isLittleEndian()); SCOPED_MEMORY_ACCESS.putShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstOffset + offset, v, !Architecture.isLittleEndian()); - offset += 2; - remaining -=2; + offset += Short.BYTES; + remaining -= Short.BYTES; } // 0...000X if (remaining == 1) { @@ -202,9 +204,9 @@ public static int contentHash(AbstractMemorySegmentImpl segment, long fromOffset return 1; } int result = 1; - final long longBytes = length & ((1L << 62) - 8); + final long longBytes = length & LONG_MASK; final long limit = fromOffset + longBytes; - for (; fromOffset < limit; fromOffset += 8) { + for (; fromOffset < limit; fromOffset += Long.BYTES) { long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian()); result = result * POWERS_OF_31[7] + ((byte) (val >>> 56)) * POWERS_OF_31[6] @@ -218,24 +220,24 @@ public static int contentHash(AbstractMemorySegmentImpl segment, long fromOffset } int remaining = (int) (length - longBytes); // 0...0X00 - if (remaining >= 4) { + if (remaining >= Integer.BYTES) { int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian()); result = result * POWERS_OF_31[3] + ((byte) (val >>> 24)) * POWERS_OF_31[2] + ((byte) (val >>> 16)) * POWERS_OF_31[1] + ((byte) (val >>> 8)) * POWERS_OF_31[0] + ((byte) val); - fromOffset += 4; - remaining -= 4; + fromOffset += Integer.BYTES; + remaining -= Integer.BYTES; } // 0...00X0 - if (remaining >= 2) { + if (remaining >= Short.BYTES) { short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + fromOffset, !Architecture.isLittleEndian()); result = result * POWERS_OF_31[1] + ((byte) (val >>> 8)) * POWERS_OF_31[0] + ((byte) val); - fromOffset += 2; - remaining -= 2; + fromOffset += Short.BYTES; + remaining -= Short.BYTES; } // 0...000X if (remaining == 1) { @@ -288,7 +290,7 @@ private static long mismatch(AbstractMemorySegmentImpl src, long srcFromOffset, long start, int length, boolean srcAndDstBytesDiffer) { int offset = 0; final int limit = length & (NATIVE_THRESHOLD_MISMATCH - 8); - for (; offset < limit; offset += 8) { + for (; offset < limit; offset += Long.BYTES) { final long s = SCOPED_MEMORY_ACCESS.getLongUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false); final long d = SCOPED_MEMORY_ACCESS.getLongUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false); if (s != d) { @@ -298,24 +300,24 @@ private static long mismatch(AbstractMemorySegmentImpl src, long srcFromOffset, int remaining = length - offset; // 0...0X00 - if (remaining >= 4) { + if (remaining >= Integer.BYTES) { final int s = SCOPED_MEMORY_ACCESS.getIntUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false); final int d = SCOPED_MEMORY_ACCESS.getIntUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false); if (s != d) { return start + offset + mismatch(s, d); } - offset += 4; - remaining -= 4; + offset += Integer.BYTES; + remaining -= Integer.BYTES; } // 0...00X0 - if (remaining >= 2) { + if (remaining >= Short.BYTES) { final short s = SCOPED_MEMORY_ACCESS.getShortUnaligned(src.sessionImpl(), src.unsafeGetBase(), src.unsafeGetOffset() + srcFromOffset + offset, false); final short d = SCOPED_MEMORY_ACCESS.getShortUnaligned(dst.sessionImpl(), dst.unsafeGetBase(), dst.unsafeGetOffset() + dstFromOffset + offset, false); if (s != d) { return start + offset + mismatch(s, d); } - offset += 2; - remaining -= 2; + offset += Short.BYTES; + remaining -= Short.BYTES; } // 0...000X if (remaining == 1) { @@ -378,6 +380,164 @@ private static long vectorizedMismatchLargeForBytes(MemorySessionImpl aSession, return ~remaining; } + //private static final HexFormat HF = HexFormat.ofDelimiter(" "); + + /** + * {@return the shortest distance beginning at the provided {@code fromOffset} + * to the encountering of a zero byte in the provided {@code segment} + * checking bytes before the {@code toOffset}} + *

+ * The method is using a heuristic method to determine if a long word contains a + * zero byte. The method might have false positives but never false negatives. + *

+ * This method is inspired by the `glibc/string/strlen.c` implementation + * + * @param segment to examine + * @param fromOffset from where examination shall begin (inclusive) + * @param toOffset to where examination shall end (exclusive) + * @throws IllegalArgumentException if the examined region contains no zero bytes + * within a length that can be accepted by a String + */ + @ForceInline + public static int strlenByte(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + // The state has to be checked explicitly for zero-length segments + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroByte(val)) { + for (int j = 0; j < Long.BYTES; j++) { + if (SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + for (; offset < toOffset; offset++) { + byte val = SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + @ForceInline + public static int strlenShort(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroShort(val)) { + for (int j = 0; j < Long.BYTES; j += Short.BYTES) { + //System.out.println("j = " + j + " : " + HF.toHexDigits(SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j))); + if (SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + // Prevent over scanning as we step by 2 + final long endScan = toOffset & ~1; // The last bit is zero + for (; offset < endScan; offset += Short.BYTES) { + short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + //System.out.println("byte offset = " + offset + " : " + HF.toHexDigits(val)); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + // The gain of using `long` wide operations for `int` is lower than for the two other `byte` and `short` variants + // so, there is only a simpler method for `int`s + @ForceInline + public static int strlenInt(AbstractMemorySegmentImpl segment, long fromOffset, long toOffset) { + long length = Math.min(toOffset - fromOffset, ArraysSupport.SOFT_MAX_ARRAY_LENGTH) & ~3; + for (int offset = 0; offset < length; offset += Integer.BYTES) { + // We are guaranteed to be aligned here so, we can use unaligned access. + int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (val == 0) { + return offset; + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + /* + Bits 63 and N * 8 (N = 1..7) of this number are zero. Call these bits + the "holes". Note that there is a hole just to the left of + each byte, with an extra at the end: + + bits: 01111110 11111110 11111110 11111110 11111110 11111110 11111110 11111111 + bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD EEEEEEEE FFFFFFFF GGGGGGGG HHHHHHHH + + The 1-bits make sure that carries propagate to the next 0-bit. + The 0-bits provide holes for carries to fall into. + */ + private static final long HIMAGIC_FOR_BYTES = 0x8080_8080_8080_8080L; + private static final long LOMAGIC_FOR_BYTES = 0x0101_0101_0101_0101L; + + private static boolean mightContainZeroByte(long l) { + return ((l - LOMAGIC_FOR_BYTES) & (~l) & HIMAGIC_FOR_BYTES) != 0; + } + + private static final long HIMAGIC_FOR_SHORTS = 0x8000_8000_8000_8000L; + private static final long LOMAGIC_FOR_SHORTS = 0x0001_0001_0001_0001L; + + static boolean mightContainZeroShort(long l) { + return ((l - LOMAGIC_FOR_SHORTS) & (~l) & HIMAGIC_FOR_SHORTS) != 0; + } + + + private static int requireWithinArraySize(long size, + AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + if (size > ArraysSupport.SOFT_MAX_ARRAY_LENGTH) { + throw stringTooLarge(segment, fromOffset, toOffset); + } + return (int) size; + } + + private static IllegalArgumentException stringTooLarge(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return new IllegalArgumentException("String too large: " + exceptionInfo(segment, fromOffset, toOffset)); + } + + private static IndexOutOfBoundsException nullNotFound(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return new IndexOutOfBoundsException("No null terminator found: " + exceptionInfo(segment, fromOffset, toOffset)); + } + + private static String exceptionInfo(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return segment + " using region [" + fromOffset + ", " + toOffset + ")"; + } + static final String PROPERTY_PATH = "java.lang.foreign.native.threshold.power."; // The returned value is in the interval [0, 2^30] diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index 78550c56136d5..c0c6f96952761 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -27,8 +27,7 @@ import jdk.internal.access.JavaLangAccess; import jdk.internal.access.SharedSecrets; -import jdk.internal.foreign.abi.SharedUtils; -import jdk.internal.util.ArraysSupport; +import jdk.internal.vm.annotation.ForceInline; import java.lang.foreign.MemorySegment; import java.nio.charset.Charset; @@ -44,7 +43,8 @@ public final class StringSupport { private StringSupport() {} - public static String read(MemorySegment segment, long offset, Charset charset) { + @ForceInline + public static String read(AbstractMemorySegmentImpl segment, long offset, Charset charset) { return switch (CharsetKind.of(charset)) { case SINGLE_BYTE -> readByte(segment, offset, charset); case DOUBLE_BYTE -> readShort(segment, offset, charset); @@ -52,7 +52,8 @@ public static String read(MemorySegment segment, long offset, Charset charset) { }; } - public static void write(MemorySegment segment, long offset, Charset charset, String string) { + @ForceInline + public static void write(AbstractMemorySegmentImpl segment, long offset, Charset charset, String string) { switch (CharsetKind.of(charset)) { case SINGLE_BYTE -> writeByte(segment, offset, charset, string); case DOUBLE_BYTE -> writeShort(segment, offset, charset, string); @@ -60,216 +61,48 @@ public static void write(MemorySegment segment, long offset, Charset charset, St } } - private static String readByte(MemorySegment segment, long offset, Charset charset) { - long len = chunkedStrlenByte(segment, offset); - byte[] bytes = new byte[(int)len]; - MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, (int)len); + @ForceInline + private static String readByte(AbstractMemorySegmentImpl segment, long offset, Charset charset) { + final int len = SegmentBulkOperations.strlenByte(segment, offset, segment.byteSize()); + final byte[] bytes = new byte[len]; + MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); } - private static void writeByte(MemorySegment segment, long offset, Charset charset, String string) { + @ForceInline + private static void writeByte(AbstractMemorySegmentImpl segment, long offset, Charset charset, String string) { int bytes = copyBytes(string, segment, charset, offset); segment.set(JAVA_BYTE, offset + bytes, (byte)0); } - private static String readShort(MemorySegment segment, long offset, Charset charset) { - long len = chunkedStrlenShort(segment, offset); - byte[] bytes = new byte[(int)len]; - MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, (int)len); + @ForceInline + private static String readShort(AbstractMemorySegmentImpl segment, long offset, Charset charset) { + int len = SegmentBulkOperations.strlenShort(segment, offset, segment.byteSize()); + byte[] bytes = new byte[len]; + MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); } - private static void writeShort(MemorySegment segment, long offset, Charset charset, String string) { + @ForceInline + private static void writeShort(AbstractMemorySegmentImpl segment, long offset, Charset charset, String string) { int bytes = copyBytes(string, segment, charset, offset); segment.set(JAVA_SHORT_UNALIGNED, offset + bytes, (short)0); } - private static String readInt(MemorySegment segment, long offset, Charset charset) { - long len = strlenInt(segment, offset); - byte[] bytes = new byte[(int)len]; - MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, (int)len); + @ForceInline + private static String readInt(AbstractMemorySegmentImpl segment, long offset, Charset charset) { + int len = SegmentBulkOperations.strlenInt(segment, offset, segment.byteSize()); + byte[] bytes = new byte[len]; + MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); } - private static void writeInt(MemorySegment segment, long offset, Charset charset, String string) { + @ForceInline + private static void writeInt(AbstractMemorySegmentImpl segment, long offset, Charset charset, String string) { int bytes = copyBytes(string, segment, charset, offset); segment.set(JAVA_INT_UNALIGNED, offset + bytes, 0); } - /** - * {@return the shortest distance beginning at the provided {@code start} - * to the encountering of a zero byte in the provided {@code segment}} - *

- * The method divides the region of interest into three distinct regions: - *

- *

- * The body is using a heuristic method to determine if a long word - * contains a zero byte. The method might have false positives but - * never false negatives. - *

- * This method is inspired by the `glibc/string/strlen.c` implementation - * - * @param segment to examine - * @param start from where examination shall begin - * @throws IllegalArgumentException if the examined region contains no zero bytes - * within a length that can be accepted by a String - */ - public static int chunkedStrlenByte(MemorySegment segment, long start) { - - // Handle the first unaligned "head" bytes separately - int headCount = (int)SharedUtils.remainsToAlignment(segment.address() + start, Long.BYTES); - - int offset = 0; - for (; offset < headCount; offset++) { - byte curr = segment.get(JAVA_BYTE, start + offset); - if (curr == 0) { - return offset; - } - } - - // We are now on a long-aligned boundary so this is the "body" - int bodyCount = bodyCount(segment.byteSize() - start - headCount); - - for (; offset < bodyCount; offset += Long.BYTES) { - // We know we are `long` aligned so, we can save on alignment checking here - long curr = segment.get(JAVA_LONG_UNALIGNED, start + offset); - // Is this a candidate? - if (mightContainZeroByte(curr)) { - for (int j = 0; j < 8; j++) { - if (segment.get(JAVA_BYTE, start + offset + j) == 0) { - return offset + j; - } - } - } - } - - // Handle the "tail" - return requireWithinArraySize((long) offset + strlenByte(segment, start + offset)); - } - - /* Bits 63 and N * 8 (N = 1..7) of this number are zero. Call these bits - the "holes". Note that there is a hole just to the left of - each byte, with an extra at the end: - - bits: 01111110 11111110 11111110 11111110 11111110 11111110 11111110 11111111 - bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD EEEEEEEE FFFFFFFF GGGGGGGG HHHHHHHH - - The 1-bits make sure that carries propagate to the next 0-bit. - The 0-bits provide holes for carries to fall into. - */ - private static final long HIMAGIC_FOR_BYTES = 0x8080_8080_8080_8080L; - private static final long LOMAGIC_FOR_BYTES = 0x0101_0101_0101_0101L; - - static boolean mightContainZeroByte(long l) { - return ((l - LOMAGIC_FOR_BYTES) & (~l) & HIMAGIC_FOR_BYTES) != 0; - } - - private static final long HIMAGIC_FOR_SHORTS = 0x8000_8000_8000_8000L; - private static final long LOMAGIC_FOR_SHORTS = 0x0001_0001_0001_0001L; - - static boolean mightContainZeroShort(long l) { - return ((l - LOMAGIC_FOR_SHORTS) & (~l) & HIMAGIC_FOR_SHORTS) != 0; - } - - static int requireWithinArraySize(long size) { - if (size > ArraysSupport.SOFT_MAX_ARRAY_LENGTH) { - throw newIaeStringTooLarge(); - } - return (int) size; - } - - static int bodyCount(long remaining) { - return (int) Math.min( - // Make sure we do not wrap around - Integer.MAX_VALUE - Long.BYTES, - // Remaining bytes to consider - remaining) - & -Long.BYTES; // Mask 0xFFFFFFF8 - } - - private static int strlenByte(MemorySegment segment, long start) { - for (int offset = 0; offset < ArraysSupport.SOFT_MAX_ARRAY_LENGTH; offset += 1) { - byte curr = segment.get(JAVA_BYTE, start + offset); - if (curr == 0) { - return offset; - } - } - throw newIaeStringTooLarge(); - } - - /** - * {@return the shortest distance beginning at the provided {@code start} - * to the encountering of a zero short in the provided {@code segment}} - *

- * Note: The inspected region must be short aligned. - * - * @see #chunkedStrlenByte(MemorySegment, long) for more information - * - * @param segment to examine - * @param start from where examination shall begin - * @throws IllegalArgumentException if the examined region contains no zero shorts - * within a length that can be accepted by a String - */ - public static int chunkedStrlenShort(MemorySegment segment, long start) { - - // Handle the first unaligned "head" bytes separately - int headCount = (int)SharedUtils.remainsToAlignment(segment.address() + start, Long.BYTES); - - int offset = 0; - for (; offset < headCount; offset += Short.BYTES) { - short curr = segment.get(JAVA_SHORT_UNALIGNED, start + offset); - if (curr == 0) { - return offset; - } - } - - // We are now on a long-aligned boundary so this is the "body" - int bodyCount = bodyCount(segment.byteSize() - start - headCount); - - for (; offset < bodyCount; offset += Long.BYTES) { - // We know we are `long` aligned so, we can save on alignment checking here - long curr = segment.get(JAVA_LONG_UNALIGNED, start + offset); - // Is this a candidate? - if (mightContainZeroShort(curr)) { - for (int j = 0; j < Long.BYTES; j += Short.BYTES) { - if (segment.get(JAVA_SHORT_UNALIGNED, start + offset + j) == 0) { - return offset + j; - } - } - } - } - - // Handle the "tail" - return requireWithinArraySize((long) offset + strlenShort(segment, start + offset)); - } - - private static int strlenShort(MemorySegment segment, long start) { - for (int offset = 0; offset < ArraysSupport.SOFT_MAX_ARRAY_LENGTH; offset += Short.BYTES) { - short curr = segment.get(JAVA_SHORT_UNALIGNED, start + offset); - if (curr == (short)0) { - return offset; - } - } - throw newIaeStringTooLarge(); - } - - // The gain of using `long` wide operations for `int` is lower than for the two other `byte` and `short` variants - // so, there is only one method for ints. - public static int strlenInt(MemorySegment segment, long start) { - for (int offset = 0; offset < ArraysSupport.SOFT_MAX_ARRAY_LENGTH; offset += Integer.BYTES) { - // We are guaranteed to be aligned here so, we can use unaligned access. - int curr = segment.get(JAVA_INT_UNALIGNED, start + offset); - if (curr == 0) { - return offset; - } - } - throw newIaeStringTooLarge(); - } - public enum CharsetKind { SINGLE_BYTE(1), DOUBLE_BYTE(2), @@ -323,9 +156,4 @@ public static int copyBytes(String string, MemorySegment segment, Charset charse public static void copyToSegmentRaw(String string, MemorySegment segment, long offset) { JAVA_LANG_ACCESS.copyToSegmentRaw(string, segment, offset); } - - private static IllegalArgumentException newIaeStringTooLarge() { - return new IllegalArgumentException("String too large"); - } - } diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index 4caef6fbd09b4..e84d4f0f6d1db 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -35,10 +35,13 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.HexFormat; import java.util.List; import java.util.Random; import java.util.function.UnaryOperator; +import jdk.internal.foreign.AbstractMemorySegmentImpl; +import jdk.internal.foreign.SegmentBulkOperations; import jdk.internal.foreign.StringSupport; import org.testng.annotations.*; @@ -87,7 +90,6 @@ public void testStrings(String testString) { } } - @Test(dataProvider = "strings") public void testStringsHeap(String testString) { for (Charset charset : singleByteCharsets()) { @@ -198,8 +200,21 @@ public void testOffset(String testString) { try (arena) { MemorySegment inSegment = arena.allocateFrom(testString, charset); for (int i = 0; i < 3; i++) { + String expected = testString.substring(i); + String actual = inSegment.getString(i, charset); - assertEquals(actual, testString.substring(i)); + if (!actual.equals(expected)) { + System.out.println("i = " + i + " failed"); + System.out.println("testString = " + testString); + System.out.println("actual = '" + actual + "'"); + //System.out.println("actual = " + HF.formatHex(actual.getBytes(StandardCharsets.UTF_8))); + System.out.println("substr = '" + testString.substring(1) + "'"); + //System.out.println("substr = " + HF.formatHex(expected.getBytes(StandardCharsets.UTF_8))); + System.out.flush(); + } else { + System.out.println("i = " + i + " ok"); + } + assertEquals(actual, expected); } } } @@ -271,7 +286,7 @@ public void chunked_strlen_byte() { } segment.setAtIndex(JAVA_BYTE, len, (byte) 0); for (int j = 0; j < len; j++) { - int actual = StringSupport.chunkedStrlenByte(segment, j); + int actual = SegmentBulkOperations.strlenByte((AbstractMemorySegmentImpl) segment, j, segment.byteSize()); assertEquals(actual, len - j); } } @@ -295,7 +310,7 @@ public void chunked_strlen_short() { } segment.setAtIndex(JAVA_SHORT, len, (short) 0); for (int j = 0; j < len; j++) { - int actual = StringSupport.chunkedStrlenShort(segment, j * Short.BYTES); + int actual = SegmentBulkOperations.strlenShort((AbstractMemorySegmentImpl) segment, j * Short.BYTES, segment.byteSize()); assertEquals(actual, (len - j) * Short.BYTES); } } @@ -319,7 +334,7 @@ public void strlen_int() { } segment.setAtIndex(JAVA_INT, len, 0); for (int j = 0; j < len; j++) { - int actual = StringSupport.strlenInt(segment, j * Integer.BYTES); + int actual = SegmentBulkOperations.strlenInt((AbstractMemorySegmentImpl) segment, j * Integer.BYTES, segment.byteSize()); assertEquals(actual, (len - j) * Integer.BYTES); } } diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java index 2db15bfe2652d..8746a6588ee31 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java @@ -22,6 +22,8 @@ */ package org.openjdk.bench.java.lang.foreign; +import jdk.internal.foreign.AbstractMemorySegmentImpl; +import jdk.internal.foreign.SegmentBulkOperations; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -50,13 +52,14 @@ @Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 3, jvmArgs = {"--add-exports=java.base/jdk.internal.foreign=ALL-UNNAMED", "--enable-native-access=ALL-UNNAMED", "--enable-preview"}) +@Fork(value = 3, jvmArgs = {"--add-exports=java.base/jdk.internal.foreign=ALL-UNNAMED", + "--enable-native-access=ALL-UNNAMED"}) public class InternalStrLen { - private MemorySegment singleByteSegment; - private MemorySegment singleByteSegmentMisaligned; - private MemorySegment doubleByteSegment; - private MemorySegment quadByteSegment; + private AbstractMemorySegmentImpl singleByteSegment; + private AbstractMemorySegmentImpl singleByteSegmentMisaligned; + private AbstractMemorySegmentImpl doubleByteSegment; + private AbstractMemorySegmentImpl quadByteSegment; @Param({"1", "4", "16", "251", "1024"}) int size; @@ -64,10 +67,9 @@ public class InternalStrLen { @Setup public void setup() { var arena = Arena.ofAuto(); - singleByteSegment = arena.allocate((size + 1L) * Byte.BYTES); - singleByteSegmentMisaligned = arena.allocate((size + 1L) * Byte.BYTES); - doubleByteSegment = arena.allocate((size + 1L) * Short.BYTES); - quadByteSegment = arena.allocate((size + 1L) * Integer.BYTES); + singleByteSegment = (AbstractMemorySegmentImpl) arena.allocate((size + 1L) * Byte.BYTES); + doubleByteSegment = (AbstractMemorySegmentImpl) arena.allocate((size + 1L) * Short.BYTES); + quadByteSegment = (AbstractMemorySegmentImpl) arena.allocate((size + 1L) * Integer.BYTES); Stream.of(singleByteSegment, doubleByteSegment, quadByteSegment) .forEach(s -> IntStream.range(0, (int) s.byteSize() - 1) .forEach(i -> s.set( @@ -78,7 +80,7 @@ public void setup() { singleByteSegment.set(ValueLayout.JAVA_BYTE, singleByteSegment.byteSize() - Byte.BYTES, (byte) 0); doubleByteSegment.set(ValueLayout.JAVA_SHORT, doubleByteSegment.byteSize() - Short.BYTES, (short) 0); quadByteSegment.set(ValueLayout.JAVA_INT, quadByteSegment.byteSize() - Integer.BYTES, 0); - singleByteSegmentMisaligned = arena.allocate(singleByteSegment.byteSize() + 1). + singleByteSegmentMisaligned = (AbstractMemorySegmentImpl) arena.allocate(singleByteSegment.byteSize() + 1). asSlice(1); MemorySegment.copy(singleByteSegment, 0, singleByteSegmentMisaligned, 0, singleByteSegment.byteSize()); } @@ -105,17 +107,17 @@ public int elementQuad() { @Benchmark public int chunkedSingle() { - return chunkedStrlenByte(singleByteSegment, 0); + return SegmentBulkOperations.strlenByte(singleByteSegment, 0, singleByteSegment.byteSize()); } @Benchmark public int chunkedSingleMisaligned() { - return chunkedStrlenByte(singleByteSegmentMisaligned, 0); + return SegmentBulkOperations.strlenByte(singleByteSegmentMisaligned, 0, singleByteSegment.byteSize()); } @Benchmark public int chunkedDouble() { - return chunkedStrlenShort(doubleByteSegment, 0); + return SegmentBulkOperations.strlenShort(doubleByteSegment, 0, doubleByteSegment.byteSize()); } @Benchmark From 58e3d35bb9f6fe5867fded53546d8434ef620a8a Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 08:00:49 +0100 Subject: [PATCH 02/11] Fix benchmark --- .../org/openjdk/bench/java/lang/foreign/InternalStrLen.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java index 8746a6588ee31..a875f6ac97b49 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java @@ -122,7 +122,7 @@ public int chunkedDouble() { @Benchmark public int changedElementQuad() { - return strlenInt(quadByteSegment, 0); + return SegmentBulkOperations.strlenInt(quadByteSegment, 0, quadByteSegment.byteSize()); } // These are the legacy methods From 50b60b436f60e80079ebe2d90fff2abe283756fd Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 08:43:26 +0100 Subject: [PATCH 03/11] Add int method --- .../foreign/SegmentBulkOperations.java | 44 ++++++++++++++++--- test/jdk/java/foreign/TestStringEncoding.java | 12 ----- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java index f9ba0c240442b..5d16f952be957 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java +++ b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java @@ -449,7 +449,6 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); if (mightContainZeroShort(val)) { for (int j = 0; j < Long.BYTES; j += Short.BYTES) { - //System.out.println("j = " + j + " : " + HF.toHexDigits(SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j))); if (SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); } @@ -461,7 +460,6 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, final long endScan = toOffset & ~1; // The last bit is zero for (; offset < endScan; offset += Short.BYTES) { short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - //System.out.println("byte offset = " + offset + " : " + HF.toHexDigits(val)); if (val == 0) { return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); } @@ -469,8 +467,44 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, throw nullNotFound(segment, fromOffset, toOffset); } - // The gain of using `long` wide operations for `int` is lower than for the two other `byte` and `short` variants - // so, there is only a simpler method for `int`s + @ForceInline + public static int strlenInt(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroShort(val)) { + for (int j = 0; j < Long.BYTES; j += Integer.BYTES) { + if (SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + // Prevent over scanning as we step by 4 + final long endScan = toOffset & ~3; // The last two bit are zero + for (; offset < endScan; offset += Integer.BYTES) { + int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + +/* // The gain of using `long` wide operations for `int` is lower than for the two other + // `byte` and `short` variants so, there is only a simpler method for `int`s @ForceInline public static int strlenInt(AbstractMemorySegmentImpl segment, long fromOffset, long toOffset) { long length = Math.min(toOffset - fromOffset, ArraysSupport.SOFT_MAX_ARRAY_LENGTH) & ~3; @@ -482,7 +516,7 @@ public static int strlenInt(AbstractMemorySegmentImpl segment, long fromOffset, } } throw nullNotFound(segment, fromOffset, toOffset); - } + }*/ /* Bits 63 and N * 8 (N = 1..7) of this number are zero. Call these bits diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index e84d4f0f6d1db..bee525b49bd1e 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -201,19 +201,7 @@ public void testOffset(String testString) { MemorySegment inSegment = arena.allocateFrom(testString, charset); for (int i = 0; i < 3; i++) { String expected = testString.substring(i); - String actual = inSegment.getString(i, charset); - if (!actual.equals(expected)) { - System.out.println("i = " + i + " failed"); - System.out.println("testString = " + testString); - System.out.println("actual = '" + actual + "'"); - //System.out.println("actual = " + HF.formatHex(actual.getBytes(StandardCharsets.UTF_8))); - System.out.println("substr = '" + testString.substring(1) + "'"); - //System.out.println("substr = " + HF.formatHex(expected.getBytes(StandardCharsets.UTF_8))); - System.out.flush(); - } else { - System.out.println("i = " + i + " ok"); - } assertEquals(actual, expected); } } From b6b464442311e1835f212f2484e6ad6d09cecd7f Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 08:57:18 +0100 Subject: [PATCH 04/11] Clean up --- .../classes/jdk/internal/foreign/SegmentBulkOperations.java | 2 -- test/jdk/java/foreign/TestStringEncoding.java | 2 +- .../org/openjdk/bench/java/lang/foreign/InternalStrLen.java | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java index 5d16f952be957..a1d78ede2d198 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java +++ b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java @@ -33,8 +33,6 @@ import java.lang.foreign.MemorySegment; -import static java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED; - /** * This class contains optimized bulk operation methods that operate on one or several * memory segments. diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index bee525b49bd1e..8709a92cab466 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java index a875f6ac97b49..68cd0d36d90c4 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 777458ad258c3ee782c27b79b2e9c922d4349050 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 10:37:22 +0100 Subject: [PATCH 05/11] Move back methods to StringSupport --- .../foreign/SegmentBulkOperations.java | 192 ------------------ .../jdk/internal/foreign/StringSupport.java | 187 ++++++++++++++++- test/jdk/java/foreign/TestStringEncoding.java | 6 +- .../java/lang/foreign/InternalStrLen.java | 9 +- 4 files changed, 191 insertions(+), 203 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java index a1d78ede2d198..068db1bf593c6 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java +++ b/src/java.base/share/classes/jdk/internal/foreign/SegmentBulkOperations.java @@ -378,198 +378,6 @@ private static long vectorizedMismatchLargeForBytes(MemorySessionImpl aSession, return ~remaining; } - //private static final HexFormat HF = HexFormat.ofDelimiter(" "); - - /** - * {@return the shortest distance beginning at the provided {@code fromOffset} - * to the encountering of a zero byte in the provided {@code segment} - * checking bytes before the {@code toOffset}} - *

- * The method is using a heuristic method to determine if a long word contains a - * zero byte. The method might have false positives but never false negatives. - *

- * This method is inspired by the `glibc/string/strlen.c` implementation - * - * @param segment to examine - * @param fromOffset from where examination shall begin (inclusive) - * @param toOffset to where examination shall end (exclusive) - * @throws IllegalArgumentException if the examined region contains no zero bytes - * within a length that can be accepted by a String - */ - @ForceInline - public static int strlenByte(final AbstractMemorySegmentImpl segment, - final long fromOffset, - final long toOffset) { - final long length = toOffset - fromOffset; - segment.checkBounds(fromOffset, length); - if (length == 0) { - // The state has to be checked explicitly for zero-length segments - segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); - } - final long longBytes = length & LONG_MASK; - final long longLimit = fromOffset + longBytes; - long offset = fromOffset; - for (; offset < longLimit; offset += Long.BYTES) { - long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (mightContainZeroByte(val)) { - for (int j = 0; j < Long.BYTES; j++) { - if (SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); - } - } - } - } - // Handle the tail - for (; offset < toOffset; offset++) { - byte val = SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset); - if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); - } - } - throw nullNotFound(segment, fromOffset, toOffset); - } - - @ForceInline - public static int strlenShort(final AbstractMemorySegmentImpl segment, - final long fromOffset, - final long toOffset) { - final long length = toOffset - fromOffset; - segment.checkBounds(fromOffset, length); - if (length == 0) { - segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); - } - final long longBytes = length & LONG_MASK; - final long longLimit = fromOffset + longBytes; - long offset = fromOffset; - for (; offset < longLimit; offset += Long.BYTES) { - long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (mightContainZeroShort(val)) { - for (int j = 0; j < Long.BYTES; j += Short.BYTES) { - if (SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); - } - } - } - } - // Handle the tail - // Prevent over scanning as we step by 2 - final long endScan = toOffset & ~1; // The last bit is zero - for (; offset < endScan; offset += Short.BYTES) { - short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); - } - } - throw nullNotFound(segment, fromOffset, toOffset); - } - - @ForceInline - public static int strlenInt(final AbstractMemorySegmentImpl segment, - final long fromOffset, - final long toOffset) { - final long length = toOffset - fromOffset; - segment.checkBounds(fromOffset, length); - if (length == 0) { - segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); - } - final long longBytes = length & LONG_MASK; - final long longLimit = fromOffset + longBytes; - long offset = fromOffset; - for (; offset < longLimit; offset += Long.BYTES) { - long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (mightContainZeroShort(val)) { - for (int j = 0; j < Long.BYTES; j += Integer.BYTES) { - if (SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); - } - } - } - } - // Handle the tail - // Prevent over scanning as we step by 4 - final long endScan = toOffset & ~3; // The last two bit are zero - for (; offset < endScan; offset += Integer.BYTES) { - int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); - } - } - throw nullNotFound(segment, fromOffset, toOffset); - } - - -/* // The gain of using `long` wide operations for `int` is lower than for the two other - // `byte` and `short` variants so, there is only a simpler method for `int`s - @ForceInline - public static int strlenInt(AbstractMemorySegmentImpl segment, long fromOffset, long toOffset) { - long length = Math.min(toOffset - fromOffset, ArraysSupport.SOFT_MAX_ARRAY_LENGTH) & ~3; - for (int offset = 0; offset < length; offset += Integer.BYTES) { - // We are guaranteed to be aligned here so, we can use unaligned access. - int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (val == 0) { - return offset; - } - } - throw nullNotFound(segment, fromOffset, toOffset); - }*/ - - /* - Bits 63 and N * 8 (N = 1..7) of this number are zero. Call these bits - the "holes". Note that there is a hole just to the left of - each byte, with an extra at the end: - - bits: 01111110 11111110 11111110 11111110 11111110 11111110 11111110 11111111 - bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD EEEEEEEE FFFFFFFF GGGGGGGG HHHHHHHH - - The 1-bits make sure that carries propagate to the next 0-bit. - The 0-bits provide holes for carries to fall into. - */ - private static final long HIMAGIC_FOR_BYTES = 0x8080_8080_8080_8080L; - private static final long LOMAGIC_FOR_BYTES = 0x0101_0101_0101_0101L; - - private static boolean mightContainZeroByte(long l) { - return ((l - LOMAGIC_FOR_BYTES) & (~l) & HIMAGIC_FOR_BYTES) != 0; - } - - private static final long HIMAGIC_FOR_SHORTS = 0x8000_8000_8000_8000L; - private static final long LOMAGIC_FOR_SHORTS = 0x0001_0001_0001_0001L; - - static boolean mightContainZeroShort(long l) { - return ((l - LOMAGIC_FOR_SHORTS) & (~l) & HIMAGIC_FOR_SHORTS) != 0; - } - - - private static int requireWithinArraySize(long size, - AbstractMemorySegmentImpl segment, - long fromOffset, - long toOffset) { - if (size > ArraysSupport.SOFT_MAX_ARRAY_LENGTH) { - throw stringTooLarge(segment, fromOffset, toOffset); - } - return (int) size; - } - - private static IllegalArgumentException stringTooLarge(AbstractMemorySegmentImpl segment, - long fromOffset, - long toOffset) { - return new IllegalArgumentException("String too large: " + exceptionInfo(segment, fromOffset, toOffset)); - } - - private static IndexOutOfBoundsException nullNotFound(AbstractMemorySegmentImpl segment, - long fromOffset, - long toOffset) { - return new IndexOutOfBoundsException("No null terminator found: " + exceptionInfo(segment, fromOffset, toOffset)); - } - - private static String exceptionInfo(AbstractMemorySegmentImpl segment, - long fromOffset, - long toOffset) { - return segment + " using region [" + fromOffset + ", " + toOffset + ")"; - } - static final String PROPERTY_PATH = "java.lang.foreign.native.threshold.power."; // The returned value is in the interval [0, 2^30] diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index c0c6f96952761..1efbc7677024a 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -27,6 +27,9 @@ import jdk.internal.access.JavaLangAccess; import jdk.internal.access.SharedSecrets; +import jdk.internal.misc.ScopedMemoryAccess; +import jdk.internal.util.Architecture; +import jdk.internal.util.ArraysSupport; import jdk.internal.vm.annotation.ForceInline; import java.lang.foreign.MemorySegment; @@ -39,7 +42,9 @@ */ public final class StringSupport { - static final JavaLangAccess JAVA_LANG_ACCESS = SharedSecrets.getJavaLangAccess(); + private static final JavaLangAccess JAVA_LANG_ACCESS = SharedSecrets.getJavaLangAccess(); + private static final ScopedMemoryAccess SCOPED_MEMORY_ACCESS = ScopedMemoryAccess.getScopedMemoryAccess(); + private static final long LONG_MASK = ~7L; // The last three bits are zero private StringSupport() {} @@ -63,7 +68,7 @@ public static void write(AbstractMemorySegmentImpl segment, long offset, Charset @ForceInline private static String readByte(AbstractMemorySegmentImpl segment, long offset, Charset charset) { - final int len = SegmentBulkOperations.strlenByte(segment, offset, segment.byteSize()); + final int len = strlenByte(segment, offset, segment.byteSize()); final byte[] bytes = new byte[len]; MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); @@ -77,7 +82,7 @@ private static void writeByte(AbstractMemorySegmentImpl segment, long offset, Ch @ForceInline private static String readShort(AbstractMemorySegmentImpl segment, long offset, Charset charset) { - int len = SegmentBulkOperations.strlenShort(segment, offset, segment.byteSize()); + int len = strlenShort(segment, offset, segment.byteSize()); byte[] bytes = new byte[len]; MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); @@ -91,7 +96,7 @@ private static void writeShort(AbstractMemorySegmentImpl segment, long offset, C @ForceInline private static String readInt(AbstractMemorySegmentImpl segment, long offset, Charset charset) { - int len = SegmentBulkOperations.strlenInt(segment, offset, segment.byteSize()); + int len = strlenInt(segment, offset, segment.byteSize()); byte[] bytes = new byte[len]; MemorySegment.copy(segment, JAVA_BYTE, offset, bytes, 0, len); return new String(bytes, charset); @@ -103,6 +108,180 @@ private static void writeInt(AbstractMemorySegmentImpl segment, long offset, Cha segment.set(JAVA_INT_UNALIGNED, offset + bytes, 0); } + /** + * {@return the shortest distance beginning at the provided {@code fromOffset} + * to the encountering of a zero byte in the provided {@code segment} + * checking bytes before the {@code toOffset}} + *

+ * The method is using a heuristic method to determine if a long word contains a + * zero byte. The method might have false positives but never false negatives. + *

+ * This method is inspired by the `glibc/string/strlen.c` implementation + * + * @param segment to examine + * @param fromOffset from where examination shall begin (inclusive) + * @param toOffset to where examination shall end (exclusive) + * @throws IllegalArgumentException if the examined region contains no zero bytes + * within a length that can be accepted by a String + */ + @ForceInline + public static int strlenByte(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + // The state has to be checked explicitly for zero-length segments + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroByte(val)) { + for (int j = 0; j < Long.BYTES; j++) { + if (SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + for (; offset < toOffset; offset++) { + byte val = SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + @ForceInline + public static int strlenShort(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroShort(val)) { + for (int j = 0; j < Long.BYTES; j += Short.BYTES) { + if (SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + // Prevent over scanning as we step by 2 + final long endScan = toOffset & ~1; // The last bit is zero + for (; offset < endScan; offset += Short.BYTES) { + short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + @ForceInline + public static int strlenInt(final AbstractMemorySegmentImpl segment, + final long fromOffset, + final long toOffset) { + final long length = toOffset - fromOffset; + segment.checkBounds(fromOffset, length); + if (length == 0) { + segment.scope.checkValidState(); + throw stringTooLarge(segment, fromOffset, toOffset); + } + final long longBytes = length & LONG_MASK; + final long longLimit = fromOffset + longBytes; + long offset = fromOffset; + for (; offset < longLimit; offset += Long.BYTES) { + long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (mightContainZeroShort(val)) { + for (int j = 0; j < Long.BYTES; j += Integer.BYTES) { + if (SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { + return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + } + } + } + } + // Handle the tail + // Prevent over scanning as we step by 4 + final long endScan = toOffset & ~3; // The last two bit are zero + for (; offset < endScan; offset += Integer.BYTES) { + int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); + if (val == 0) { + return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + } + } + throw nullNotFound(segment, fromOffset, toOffset); + } + + /* + Bits 63 and N * 8 (N = 1..7) of this number are zero. Call these bits + the "holes". Note that there is a hole just to the left of + each byte, with an extra at the end: + + bits: 01111110 11111110 11111110 11111110 11111110 11111110 11111110 11111111 + bytes: AAAAAAAA BBBBBBBB CCCCCCCC DDDDDDDD EEEEEEEE FFFFFFFF GGGGGGGG HHHHHHHH + + The 1-bits make sure that carries propagate to the next 0-bit. + The 0-bits provide holes for carries to fall into. + */ + private static final long HIMAGIC_FOR_BYTES = 0x8080_8080_8080_8080L; + private static final long LOMAGIC_FOR_BYTES = 0x0101_0101_0101_0101L; + + private static boolean mightContainZeroByte(long l) { + return ((l - LOMAGIC_FOR_BYTES) & (~l) & HIMAGIC_FOR_BYTES) != 0; + } + + private static final long HIMAGIC_FOR_SHORTS = 0x8000_8000_8000_8000L; + private static final long LOMAGIC_FOR_SHORTS = 0x0001_0001_0001_0001L; + + static boolean mightContainZeroShort(long l) { + return ((l - LOMAGIC_FOR_SHORTS) & (~l) & HIMAGIC_FOR_SHORTS) != 0; + } + + + private static int requireWithinArraySize(long size, + AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + if (size > ArraysSupport.SOFT_MAX_ARRAY_LENGTH) { + throw stringTooLarge(segment, fromOffset, toOffset); + } + return (int) size; + } + + private static IllegalArgumentException stringTooLarge(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return new IllegalArgumentException("String too large: " + exceptionInfo(segment, fromOffset, toOffset)); + } + + private static IndexOutOfBoundsException nullNotFound(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return new IndexOutOfBoundsException("No null terminator found: " + exceptionInfo(segment, fromOffset, toOffset)); + } + + private static String exceptionInfo(AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { + return segment + " using region [" + fromOffset + ", " + toOffset + ")"; + } + public enum CharsetKind { SINGLE_BYTE(1), DOUBLE_BYTE(2), diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index 8709a92cab466..5a4a0897f9865 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -274,7 +274,7 @@ public void chunked_strlen_byte() { } segment.setAtIndex(JAVA_BYTE, len, (byte) 0); for (int j = 0; j < len; j++) { - int actual = SegmentBulkOperations.strlenByte((AbstractMemorySegmentImpl) segment, j, segment.byteSize()); + int actual = StringSupport.strlenByte((AbstractMemorySegmentImpl) segment, j, segment.byteSize()); assertEquals(actual, len - j); } } @@ -298,7 +298,7 @@ public void chunked_strlen_short() { } segment.setAtIndex(JAVA_SHORT, len, (short) 0); for (int j = 0; j < len; j++) { - int actual = SegmentBulkOperations.strlenShort((AbstractMemorySegmentImpl) segment, j * Short.BYTES, segment.byteSize()); + int actual = StringSupport.strlenShort((AbstractMemorySegmentImpl) segment, j * Short.BYTES, segment.byteSize()); assertEquals(actual, (len - j) * Short.BYTES); } } @@ -322,7 +322,7 @@ public void strlen_int() { } segment.setAtIndex(JAVA_INT, len, 0); for (int j = 0; j < len; j++) { - int actual = SegmentBulkOperations.strlenInt((AbstractMemorySegmentImpl) segment, j * Integer.BYTES, segment.byteSize()); + int actual = StringSupport.strlenInt((AbstractMemorySegmentImpl) segment, j * Integer.BYTES, segment.byteSize()); assertEquals(actual, (len - j) * Integer.BYTES); } } diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java index 68cd0d36d90c4..6e5f6093e3c28 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java @@ -24,6 +24,7 @@ import jdk.internal.foreign.AbstractMemorySegmentImpl; import jdk.internal.foreign.SegmentBulkOperations; +import jdk.internal.foreign.StringSupport; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -107,22 +108,22 @@ public int elementQuad() { @Benchmark public int chunkedSingle() { - return SegmentBulkOperations.strlenByte(singleByteSegment, 0, singleByteSegment.byteSize()); + return StringSupport.strlenByte(singleByteSegment, 0, singleByteSegment.byteSize()); } @Benchmark public int chunkedSingleMisaligned() { - return SegmentBulkOperations.strlenByte(singleByteSegmentMisaligned, 0, singleByteSegment.byteSize()); + return StringSupport.strlenByte(singleByteSegmentMisaligned, 0, singleByteSegment.byteSize()); } @Benchmark public int chunkedDouble() { - return SegmentBulkOperations.strlenShort(doubleByteSegment, 0, doubleByteSegment.byteSize()); + return StringSupport.strlenShort(doubleByteSegment, 0, doubleByteSegment.byteSize()); } @Benchmark public int changedElementQuad() { - return SegmentBulkOperations.strlenInt(quadByteSegment, 0, quadByteSegment.byteSize()); + return StringSupport.strlenInt(quadByteSegment, 0, quadByteSegment.byteSize()); } // These are the legacy methods From 5426849145e8d7cd0bcbb7a970dc87d2052f5186 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Fri, 29 Nov 2024 10:41:13 +0100 Subject: [PATCH 06/11] Fix imports --- test/jdk/java/foreign/TestStringEncoding.java | 2 -- .../org/openjdk/bench/java/lang/foreign/InternalStrLen.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index 5a4a0897f9865..eb27dab6375bd 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -35,13 +35,11 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.HexFormat; import java.util.List; import java.util.Random; import java.util.function.UnaryOperator; import jdk.internal.foreign.AbstractMemorySegmentImpl; -import jdk.internal.foreign.SegmentBulkOperations; import jdk.internal.foreign.StringSupport; import org.testng.annotations.*; diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java index 6e5f6093e3c28..b7867efd77109 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/InternalStrLen.java @@ -23,7 +23,6 @@ package org.openjdk.bench.java.lang.foreign; import jdk.internal.foreign.AbstractMemorySegmentImpl; -import jdk.internal.foreign.SegmentBulkOperations; import jdk.internal.foreign.StringSupport; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -46,7 +45,6 @@ import java.util.stream.Stream; import static java.lang.foreign.ValueLayout.*; -import static jdk.internal.foreign.StringSupport.*; @BenchmarkMode(Mode.AverageTime) @Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) From 641b4636abf6e7b70f256dbd577bc8df1aa0b3eb Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Mon, 2 Dec 2024 12:22:13 +0100 Subject: [PATCH 07/11] Add test for segments with odd byte at the end --- test/jdk/java/foreign/TestStringEncoding.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index eb27dab6375bd..85c3fa416f73f 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -250,6 +250,32 @@ public void segmentationFault() { } } + // This test ensures that we do not address outside the segment even though there + // are odd bytes at the end. + @Test(dataProvider = "strings") + public void offBoundaryTrailingBytes(String testString) { + if (testString.length() < 3 || !containsOnlyRegularCharacters(testString)) { + return; + } + for (var charset : standardCharsets()) { + for (var arena: arenas()) { + try (arena) { + MemorySegment strSegment = arena.allocateFrom(testString, charset); + // Add an odd byte at the end + MemorySegment inSegment = arena.allocate(strSegment.byteSize() + 1); + // Make sure there are no null-terminators so that we will try to scan + // the entire segment. + inSegment.fill((byte) 1); + for (int i = 0; i < 4; i++) { + final int offset = i; + var e = expectThrows(IndexOutOfBoundsException.class, () -> inSegment.getString(offset, charset)); + assertTrue(e.getMessage().contains("No null terminator found")); + } + } + } + } + } + private static final int TEST_LENGTH_MAX = 277; private Random deterministicRandom() { From 2a7888689d4a929789b07c1340740c1f210a87c2 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Mon, 2 Dec 2024 12:27:22 +0100 Subject: [PATCH 08/11] Rename method --- .../jdk/internal/foreign/StringSupport.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index 1efbc7677024a..a906acc9b4c51 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -143,7 +143,7 @@ public static int strlenByte(final AbstractMemorySegmentImpl segment, if (mightContainZeroByte(val)) { for (int j = 0; j < Long.BYTES; j++) { if (SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset + j - fromOffset, segment, fromOffset, toOffset); } } } @@ -152,7 +152,7 @@ public static int strlenByte(final AbstractMemorySegmentImpl segment, for (; offset < toOffset; offset++) { byte val = SCOPED_MEMORY_ACCESS.getByte(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset); if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset - fromOffset, segment, fromOffset, toOffset); } } throw nullNotFound(segment, fromOffset, toOffset); @@ -176,7 +176,7 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, if (mightContainZeroShort(val)) { for (int j = 0; j < Long.BYTES; j += Short.BYTES) { if (SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset + j - fromOffset, segment, fromOffset, toOffset); } } } @@ -187,7 +187,7 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, for (; offset < endScan; offset += Short.BYTES) { short val = SCOPED_MEMORY_ACCESS.getShortUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset - fromOffset, segment, fromOffset, toOffset); } } throw nullNotFound(segment, fromOffset, toOffset); @@ -211,7 +211,7 @@ public static int strlenInt(final AbstractMemorySegmentImpl segment, if (mightContainZeroShort(val)) { for (int j = 0; j < Long.BYTES; j += Integer.BYTES) { if (SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { - return requireWithinArraySize(offset + j - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset + j - fromOffset, segment, fromOffset, toOffset); } } } @@ -222,7 +222,7 @@ public static int strlenInt(final AbstractMemorySegmentImpl segment, for (; offset < endScan; offset += Integer.BYTES) { int val = SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); if (val == 0) { - return requireWithinArraySize(offset - fromOffset, segment, fromOffset, toOffset); + return requireWithinStringSize(offset - fromOffset, segment, fromOffset, toOffset); } } throw nullNotFound(segment, fromOffset, toOffset); @@ -254,10 +254,10 @@ static boolean mightContainZeroShort(long l) { } - private static int requireWithinArraySize(long size, - AbstractMemorySegmentImpl segment, - long fromOffset, - long toOffset) { + private static int requireWithinStringSize(long size, + AbstractMemorySegmentImpl segment, + long fromOffset, + long toOffset) { if (size > ArraysSupport.SOFT_MAX_ARRAY_LENGTH) { throw stringTooLarge(segment, fromOffset, toOffset); } From 7ab520d4e932760df8b2dc6d68f516319d484338 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Mon, 2 Dec 2024 14:05:29 +0100 Subject: [PATCH 09/11] Add zero detection method for quad words --- .../classes/jdk/internal/foreign/StringSupport.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index a906acc9b4c51..ab0c2472670fe 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -208,7 +208,7 @@ public static int strlenInt(final AbstractMemorySegmentImpl segment, long offset = fromOffset; for (; offset < longLimit; offset += Long.BYTES) { long val = SCOPED_MEMORY_ACCESS.getLongUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset, !Architecture.isLittleEndian()); - if (mightContainZeroShort(val)) { + if (mightContainZeroInt(val)) { for (int j = 0; j < Long.BYTES; j += Integer.BYTES) { if (SCOPED_MEMORY_ACCESS.getIntUnaligned(segment.sessionImpl(), segment.unsafeGetBase(), segment.unsafeGetOffset() + offset + j, !Architecture.isLittleEndian()) == 0) { return requireWithinStringSize(offset + j - fromOffset, segment, fromOffset, toOffset); @@ -253,6 +253,13 @@ static boolean mightContainZeroShort(long l) { return ((l - LOMAGIC_FOR_SHORTS) & (~l) & HIMAGIC_FOR_SHORTS) != 0; } + private static final long HIMAGIC_FOR_INTS = 0x8000_0000_8000_0000L; + private static final long LOMAGIC_FOR_INTS = 0x0000_0001_0000_0001L; + + static boolean mightContainZeroInt(long l) { + return ((l - LOMAGIC_FOR_INTS) & (~l) & HIMAGIC_FOR_INTS) != 0; + } + private static int requireWithinStringSize(long size, AbstractMemorySegmentImpl segment, From 72b66bb569635a54bdf4ebd0ba00d02ac6861668 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Mon, 2 Dec 2024 14:42:53 +0100 Subject: [PATCH 10/11] Improve docs --- .../share/classes/jdk/internal/foreign/StringSupport.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index ab0c2472670fe..c4bbc9392ee4e 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -109,9 +109,9 @@ private static void writeInt(AbstractMemorySegmentImpl segment, long offset, Cha } /** - * {@return the shortest distance beginning at the provided {@code fromOffset} - * to the encountering of a zero byte in the provided {@code segment} - * checking bytes before the {@code toOffset}} + * {@return the index of the first zero byte beginning at the provided + * {@code fromOffset} to the encountering of a zero byte in the provided + * {@code segment} checking bytes before the {@code toOffset}} *

* The method is using a heuristic method to determine if a long word contains a * zero byte. The method might have false positives but never false negatives. From ae1acf5f05c5bfe00cfab9c455a00d9b0ecd8cb5 Mon Sep 17 00:00:00 2001 From: Per Minborg Date: Mon, 2 Dec 2024 16:36:34 +0100 Subject: [PATCH 11/11] Add test and change exception type for empty segments --- .../jdk/internal/foreign/StringSupport.java | 6 +++--- test/jdk/java/foreign/TestStringEncoding.java | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java index c4bbc9392ee4e..8f182f3b33845 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java +++ b/src/java.base/share/classes/jdk/internal/foreign/StringSupport.java @@ -133,7 +133,7 @@ public static int strlenByte(final AbstractMemorySegmentImpl segment, if (length == 0) { // The state has to be checked explicitly for zero-length segments segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); + throw nullNotFound(segment, fromOffset, toOffset); } final long longBytes = length & LONG_MASK; final long longLimit = fromOffset + longBytes; @@ -166,7 +166,7 @@ public static int strlenShort(final AbstractMemorySegmentImpl segment, segment.checkBounds(fromOffset, length); if (length == 0) { segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); + throw nullNotFound(segment, fromOffset, toOffset); } final long longBytes = length & LONG_MASK; final long longLimit = fromOffset + longBytes; @@ -201,7 +201,7 @@ public static int strlenInt(final AbstractMemorySegmentImpl segment, segment.checkBounds(fromOffset, length); if (length == 0) { segment.scope.checkValidState(); - throw stringTooLarge(segment, fromOffset, toOffset); + throw nullNotFound(segment, fromOffset, toOffset); } final long longBytes = length & LONG_MASK; final long longLimit = fromOffset + longBytes; diff --git a/test/jdk/java/foreign/TestStringEncoding.java b/test/jdk/java/foreign/TestStringEncoding.java index 85c3fa416f73f..94732943b9d36 100644 --- a/test/jdk/java/foreign/TestStringEncoding.java +++ b/test/jdk/java/foreign/TestStringEncoding.java @@ -54,6 +54,20 @@ public class TestStringEncoding { + @Test + public void emptySegment() { + for (Charset charset : standardCharsets()) { + for (Arena arena : arenas()) { + try (arena) { + var segment = arena.allocate(0); + var e = expectThrows(IndexOutOfBoundsException.class, () -> + segment.getString(0, charset)); + assertTrue(e.getMessage().contains("No null terminator found")); + } + } + } + } + @Test(dataProvider = "strings") public void testStrings(String testString) { for (Charset charset : Charset.availableCharsets().values()) { @@ -301,6 +315,12 @@ public void chunked_strlen_byte() { int actual = StringSupport.strlenByte((AbstractMemorySegmentImpl) segment, j, segment.byteSize()); assertEquals(actual, len - j); } + // Test end offset + for (int j = 0; j < len - 1; j++) { + final long toOffset = j; + expectThrows(IndexOutOfBoundsException.class, () -> + StringSupport.strlenByte((AbstractMemorySegmentImpl) segment, 0, toOffset)); + } } } }