From baeb64699b2fa5cb4f66fb2c54b1f998758cea7d Mon Sep 17 00:00:00 2001
From: Lee Rhodes
Exceptions during update. If {@link #update(float)} throws an {@link OutOfMemoryError} before + * the item is stored, the item is not added and the sketch is unchanged. If the error happens later, while + * the sketch compresses, the item has been added and no retained item is lost: the sketch stays valid and + * compresses again on the next update. A merge may have applied part of the other sketch when it throws.
+ * * @see QuantilesAPI * * @author Edo Liberty @@ -469,6 +474,8 @@ public String toString() { public void update(final float item) { if (Float.isNaN(item)) { return; } ensureCapacity(); //first use allocates, before anything changes + final FloatBuffer fbuf = compactors.get(0).getBuffer(); + fbuf.append(item); //may grow the buffer; nothing else has changed yet if (isEmpty()) { minItem = item; maxItem = item; @@ -476,8 +483,6 @@ public void update(final float item) { if (item < minItem) { minItem = item; } if (item > maxItem) { maxItem = item; } } - final FloatBuffer fbuf = compactors.get(0).getBuffer(); - fbuf.append(item); retItems++; totalN++; if (retItems >= maxNomSize) { @@ -554,26 +559,35 @@ private static void checkK(final int k) { } } + //Each compaction moves items to the next level only after the space for them has been allocated, + //so a failed allocation never loses items. If an error interrupts compression, the counters are + //recomputed from the compactors, so the sketch stays consistent and the next update compresses again. private void compress() { if (reqDebug != null) { reqDebug.emitStartCompress(); } - for (int h = 0; h < compactors.size(); h++) { - final ReqCompactor c = compactors.get(h); - final int compRetItems = c.getBuffer().getCount(); - final int compNomCap = c.getNomCapacity(); - - if (compRetItems >= compNomCap) { - if ((h + 1) >= getNumLevels()) { //at the top? - if (reqDebug != null) { reqDebug.emitMustAddCompactor(); } - grow(); //add a level, increases maxNomSize + try { + for (int h = 0; h < compactors.size(); h++) { + final ReqCompactor c = compactors.get(h); + final int compRetItems = c.getBuffer().getCount(); + final int compNomCap = c.getNomCapacity(); + + if (compRetItems >= compNomCap) { + if ((h + 1) >= getNumLevels()) { //at the top? + if (reqDebug != null) { reqDebug.emitMustAddCompactor(); } + grow(); //add a level, increases maxNomSize + } + c.compact(cReturn, rand, compactors.get(h + 1).getBuffer()); + retItems += cReturn.deltaRetItems; + maxNomSize += cReturn.deltaNomSize; + //we specifically decided not to do lazy compression. } - final FloatBuffer promoted = c.compact(cReturn, rand); - compactors.get(h + 1).getBuffer().mergeSortIn(promoted); - retItems += cReturn.deltaRetItems; - maxNomSize += cReturn.deltaNomSize; - //we specifically decided not to do lazy compression. } + } catch (final RuntimeException | Error e) { + retItems = computeTotalRetainedItems(); + maxNomSize = computeMaxNomSize(); + throw e; + } finally { + reqSV = null; } - reqSV = null; if (reqDebug != null) { reqDebug.emitCompressDone(); } } diff --git a/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java b/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java new file mode 100644 index 000000000..b4ce0f784 --- /dev/null +++ b/src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.req; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; +import java.lang.reflect.Proxy; +import java.util.Random; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; +import org.testng.annotations.Test; + +/** + * Checks that a ReqSketch stays consistent when an error interrupts an update. + */ +public class ReqUpdateFailureTest { + private static final int K = 12; + + private static float value(final int i) { return (i * 7919) % 100_003; } + + /** + * A compaction allocates the space in the next level before it changes anything. Here the merge into the + * next level fails instead, because that buffer is not sorted. Both compactors must be left as they were. + */ + @Test + public void compactorUnchangedWhenPromotionFails() { + for (final boolean hra : new boolean[] {true, false}) { + final ReqCompactor c = new ReqCompactor((byte) 0, hra, K, null); + final FloatBuffer buf = c.getBuffer(); + for (int i = 0; i < c.getNomCapacity(); i++) { buf.append(value(i)); } + buf.sort(); + final FloatBuffer next = new FloatBuffer(4, 4, hra); + next.append(2f); + next.append(1f); //not sorted + final byte[] before = c.toByteArray(); + final int nextCount = next.getCount(); + final boolean coin = c.getCoin(); + try { + c.compact(new ReqSketch.CompactorReturn(), new Random(1), next); + fail(); + } catch (final SketchesArgumentException e) { + //expected + } + assertEquals(c.toByteArray(), before); + assertEquals(c.getCoin(), coin); + assertEquals(next.getCount(), nextCount); + } + } + + /** + * An error during compression leaves the item that started it in the sketch, with counts, bounds and + * min and max that agree with the compactors, and the sketch keeps working. + */ + @Test + public void sketchConsistentWhenCompressionFails() { + for (final boolean hra : new boolean[] {true, false}) { + final int[] calls = {0}; + final ReqDebug failOnThirdLevel = (ReqDebug) Proxy.newProxyInstance(ReqDebug.class.getClassLoader(), + new Class>[] {ReqDebug.class}, (proxy, method, args) -> { + if (method.getName().equals("emitNewCompactor") && (++calls[0] == 3)) { + throw new IllegalStateException("simulated failure"); + } + return null; + }); + final ReqSketch sk = ReqSketch.builder().setK(K).setHighRankAccuracy(hra).setReqDebug(failOnThirdLevel).build(); + int n = 0; + boolean failed = false; + while (!failed) { + final float item = -n; + try { + sk.update(item); + } catch (final IllegalStateException e) { + failed = true; + } + n++; + assertEquals(sk.getN(), n); + assertEquals(sk.getMinItem(), item); //each item is the new min + checkConsistent(sk); + } + for (int i = 0; i < 10_000; i++) { sk.update(value(i)); } + checkConsistent(sk); + final ReqSketch copy = ReqSketch.heapify(MemorySegment.ofArray(sk.toByteArray())); + assertEquals(copy.toByteArray(), sk.toByteArray()); + } + } + + private static void checkConsistent(final ReqSketch sk) { + assertEquals(sk.getNumRetained(), sk.computeTotalRetainedItems()); + assertEquals(sk.getMaxNomSize(), sk.computeMaxNomSize()); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while (it.next()) { weight += it.getWeight(); } + assertEquals(weight, sk.getN()); + } +}