diff --git a/src/main/java/org/apache/datasketches/req/FloatBuffer.java b/src/main/java/org/apache/datasketches/req/FloatBuffer.java index 1c63906b9..12d2c5bd3 100755 --- a/src/main/java/org/apache/datasketches/req/FloatBuffer.java +++ b/src/main/java/org/apache/datasketches/req/FloatBuffer.java @@ -175,7 +175,7 @@ FloatBuffer ensureCapacity(final int newCapacity) { * @param space the requested space remaining * @return this */ - private FloatBuffer ensureSpace(final int space) { + FloatBuffer ensureSpace(final int space) { if ((count_ + space) > capacity_) { final int newCap = count_ + space + delta_; ensureCapacity(newCap); diff --git a/src/main/java/org/apache/datasketches/req/ReqCompactor.java b/src/main/java/org/apache/datasketches/req/ReqCompactor.java index 71aa4dfdc..412bc3699 100644 --- a/src/main/java/org/apache/datasketches/req/ReqCompactor.java +++ b/src/main/java/org/apache/datasketches/req/ReqCompactor.java @@ -141,10 +141,14 @@ void ensureCapacity() { } /** - * Perform a compaction operation on this compactor - * @return the array of items to be promoted to the next level compactor + * Perform a compaction operation on this compactor and merge the promoted items into the given + * buffer of the next level. The space in that buffer is allocated before anything changes, so an + * allocation failure leaves both compactors as they were. + * @param cReturn receives the changes in retained items and nominal capacity + * @param rand the source of the random coin flips + * @param nextBuf the sorted buffer of the next level compactor, which receives the promoted items */ - FloatBuffer compact(final CompactorReturn cReturn, final Random rand) { + void compact(final CompactorReturn cReturn, final Random rand, final FloatBuffer nextBuf) { if (reqDebug != null) { reqDebug.emitCompactingStart(lgWeight); } final int startRetItems = buf.getCount(); final int startNomCap = getNomCapacity(); @@ -154,24 +158,27 @@ FloatBuffer compact(final CompactorReturn cReturn, final Random rand) { final int compactionStart = (int) (compactionRange & 0xFFFF_FFFFL); //low 32 final int compactionEnd = (int) (compactionRange >>> 32); //high 32 assert (compactionEnd - compactionStart) >= 2; + nextBuf.ensureSpace((compactionEnd - compactionStart) / 2); //allocate before anything changes - if ((state & 1L) == 1L) { coin = !coin; } //if numCompactions odd, flip coin; - else { coin = rand.nextBoolean(); } //random coin flip + final boolean newCoin; + if ((state & 1L) == 1L) { newCoin = !coin; } //if numCompactions odd, flip coin; + else { newCoin = rand.nextBoolean(); } //random coin flip - final FloatBuffer promote = buf.getEvensOrOdds(compactionStart, compactionEnd, coin); + final FloatBuffer promote = buf.getEvensOrOdds(compactionStart, compactionEnd, newCoin); if (reqDebug != null) { reqDebug.emitCompactionDetail(compactionStart, compactionEnd, secsToCompact, - promote.getCount(), coin); + promote.getCount(), newCoin); } + nextBuf.mergeSortIn(promote); //the space was allocated above + coin = newCoin; buf.trimCount(buf.getCount() - (compactionEnd - compactionStart)); state += 1; - ensureEnoughSections(); cReturn.deltaRetItems = (buf.getCount() - startRetItems) + promote.getCount(); + ensureEnoughSections(); cReturn.deltaNomSize = getNomCapacity() - startNomCap; if (reqDebug != null) { reqDebug.emitCompactionDone(lgWeight); } - return promote; } //End Compact /** @@ -262,10 +269,10 @@ private boolean ensureEnoughSections() { && (sectionSize > MIN_K) && ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K)) { + buf.ensureCapacity(2 * NOM_CAP_MULT * (numSections << 1) * ne); //allocate before anything changes sectionSizeFlt = szf; sectionSize = ne; numSections <<= 1; - buf.ensureCapacity(2 * getNomCapacity()); if (reqDebug != null) { reqDebug.emitAdjSecSizeNumSec(lgWeight); } return true; } diff --git a/src/main/java/org/apache/datasketches/req/ReqSketch.java b/src/main/java/org/apache/datasketches/req/ReqSketch.java index 0bf58d7f0..979161490 100644 --- a/src/main/java/org/apache/datasketches/req/ReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/ReqSketch.java @@ -80,6 +80,11 @@ * not when the sketch is built. Call {@link #ensureCapacity()} right after building the sketch to allocate * the storage up front instead. Results and the serialized form are the same either way.
* + *Exceptions during update. If {@link #update(float)} throws an {@link OutOfMemoryError} before + * the item is stored, the item is not added and the sketch is unchanged. If the error happens later, while + * the sketch compresses, the item has been added and no retained item is lost: the sketch stays valid and + * compresses again on the next update. A merge may have applied part of the other sketch when it throws.
+ * * @see QuantilesAPI * * @author Edo Liberty @@ -469,6 +474,8 @@ public String toString() { public void update(final float item) { if (Float.isNaN(item)) { return; } ensureCapacity(); //first use allocates, before anything changes + final FloatBuffer fbuf = compactors.get(0).getBuffer(); + fbuf.append(item); //may grow the buffer; nothing else has changed yet if (isEmpty()) { minItem = item; maxItem = item; @@ -476,8 +483,6 @@ public void update(final float item) { if (item < minItem) { minItem = item; } if (item > maxItem) { maxItem = item; } } - final FloatBuffer fbuf = compactors.get(0).getBuffer(); - fbuf.append(item); retItems++; totalN++; if (retItems >= maxNomSize) { @@ -554,26 +559,35 @@ private static void checkK(final int k) { } } + //Each compaction moves items to the next level only after the space for them has been allocated, + //so a failed allocation never loses items. If an error interrupts compression, the counters are + //recomputed from the compactors, so the sketch stays consistent and the next update compresses again. private void compress() { if (reqDebug != null) { reqDebug.emitStartCompress(); } - for (int h = 0; h < compactors.size(); h++) { - final ReqCompactor c = compactors.get(h); - final int compRetItems = c.getBuffer().getCount(); - final int compNomCap = c.getNomCapacity(); - - if (compRetItems >= compNomCap) { - if ((h + 1) >= getNumLevels()) { //at the top? - if (reqDebug != null) { reqDebug.emitMustAddCompactor(); } - grow(); //add a level, increases maxNomSize + try { + for (int h = 0; h < compactors.size(); h++) { + final ReqCompactor c = compactors.get(h); + final int compRetItems = c.getBuffer().getCount(); + final int compNomCap = c.getNomCapacity(); + + if (compRetItems >= compNomCap) { + if ((h + 1) >= getNumLevels()) { //at the top? + if (reqDebug != null) { reqDebug.emitMustAddCompactor(); } + grow(); //add a level, increases maxNomSize + } + c.compact(cReturn, rand, compactors.get(h + 1).getBuffer()); + retItems += cReturn.deltaRetItems; + maxNomSize += cReturn.deltaNomSize; + //we specifically decided not to do lazy compression. } - final FloatBuffer promoted = c.compact(cReturn, rand); - compactors.get(h + 1).getBuffer().mergeSortIn(promoted); - retItems += cReturn.deltaRetItems; - maxNomSize += cReturn.deltaNomSize; - //we specifically decided not to do lazy compression. } + } catch (final RuntimeException | Error e) { + retItems = computeTotalRetainedItems(); + maxNomSize = computeMaxNomSize(); + throw e; + } finally { + reqSV = null; } - reqSV = null; if (reqDebug != null) { reqDebug.emitCompressDone(); } } diff --git a/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java b/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java new file mode 100644 index 000000000..b4ce0f784 --- /dev/null +++ b/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.req; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; +import java.lang.reflect.Proxy; +import java.util.Random; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; +import org.testng.annotations.Test; + +/** + * Checks that a ReqSketch stays consistent when an error interrupts an update. + */ +public class ReqUpdateFailureTest { + private static final int K = 12; + + private static float value(final int i) { return (i * 7919) % 100_003; } + + /** + * A compaction allocates the space in the next level before it changes anything. Here the merge into the + * next level fails instead, because that buffer is not sorted. Both compactors must be left as they were. + */ + @Test + public void compactorUnchangedWhenPromotionFails() { + for (final boolean hra : new boolean[] {true, false}) { + final ReqCompactor c = new ReqCompactor((byte) 0, hra, K, null); + final FloatBuffer buf = c.getBuffer(); + for (int i = 0; i < c.getNomCapacity(); i++) { buf.append(value(i)); } + buf.sort(); + final FloatBuffer next = new FloatBuffer(4, 4, hra); + next.append(2f); + next.append(1f); //not sorted + final byte[] before = c.toByteArray(); + final int nextCount = next.getCount(); + final boolean coin = c.getCoin(); + try { + c.compact(new ReqSketch.CompactorReturn(), new Random(1), next); + fail(); + } catch (final SketchesArgumentException e) { + //expected + } + assertEquals(c.toByteArray(), before); + assertEquals(c.getCoin(), coin); + assertEquals(next.getCount(), nextCount); + } + } + + /** + * An error during compression leaves the item that started it in the sketch, with counts, bounds and + * min and max that agree with the compactors, and the sketch keeps working. + */ + @Test + public void sketchConsistentWhenCompressionFails() { + for (final boolean hra : new boolean[] {true, false}) { + final int[] calls = {0}; + final ReqDebug failOnThirdLevel = (ReqDebug) Proxy.newProxyInstance(ReqDebug.class.getClassLoader(), + new Class>[] {ReqDebug.class}, (proxy, method, args) -> { + if (method.getName().equals("emitNewCompactor") && (++calls[0] == 3)) { + throw new IllegalStateException("simulated failure"); + } + return null; + }); + final ReqSketch sk = ReqSketch.builder().setK(K).setHighRankAccuracy(hra).setReqDebug(failOnThirdLevel).build(); + int n = 0; + boolean failed = false; + while (!failed) { + final float item = -n; + try { + sk.update(item); + } catch (final IllegalStateException e) { + failed = true; + } + n++; + assertEquals(sk.getN(), n); + assertEquals(sk.getMinItem(), item); //each item is the new min + checkConsistent(sk); + } + for (int i = 0; i < 10_000; i++) { sk.update(value(i)); } + checkConsistent(sk); + final ReqSketch copy = ReqSketch.heapify(MemorySegment.ofArray(sk.toByteArray())); + assertEquals(copy.toByteArray(), sk.toByteArray()); + } + } + + private static void checkConsistent(final ReqSketch sk) { + assertEquals(sk.getNumRetained(), sk.computeTotalRetainedItems()); + assertEquals(sk.getMaxNomSize(), sk.computeMaxNomSize()); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while (it.next()) { weight += it.getWeight(); } + assertEquals(weight, sk.getN()); + } +}