Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/org/apache/datasketches/req/FloatBuffer.java
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ FloatBuffer ensureCapacity(final int newCapacity) {
* @param space the requested space remaining
* @return this
*/
private FloatBuffer ensureSpace(final int space) {
FloatBuffer ensureSpace(final int space) {
if ((count_ + space) > capacity_) {
final int newCap = count_ + space + delta_;
ensureCapacity(newCap);
Expand Down
27 changes: 17 additions & 10 deletions src/main/java/org/apache/datasketches/req/ReqCompactor.java
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,14 @@ void ensureCapacity() {
}

/**
* Perform a compaction operation on this compactor
* @return the array of items to be promoted to the next level compactor
* Perform a compaction operation on this compactor and merge the promoted items into the given
* buffer of the next level. The space in that buffer is allocated before anything changes, so an
* allocation failure leaves both compactors as they were.
* @param cReturn receives the changes in retained items and nominal capacity
* @param rand the source of the random coin flips
* @param nextBuf the sorted buffer of the next level compactor, which receives the promoted items
*/
FloatBuffer compact(final CompactorReturn cReturn, final Random rand) {
void compact(final CompactorReturn cReturn, final Random rand, final FloatBuffer nextBuf) {
if (reqDebug != null) { reqDebug.emitCompactingStart(lgWeight); }
final int startRetItems = buf.getCount();
final int startNomCap = getNomCapacity();
Expand All @@ -154,24 +158,27 @@ FloatBuffer compact(final CompactorReturn cReturn, final Random rand) {
final int compactionStart = (int) (compactionRange & 0xFFFF_FFFFL); //low 32
final int compactionEnd = (int) (compactionRange >>> 32); //high 32
assert (compactionEnd - compactionStart) >= 2;
nextBuf.ensureSpace((compactionEnd - compactionStart) / 2); //allocate before anything changes

if ((state & 1L) == 1L) { coin = !coin; } //if numCompactions odd, flip coin;
else { coin = rand.nextBoolean(); } //random coin flip
final boolean newCoin;
if ((state & 1L) == 1L) { newCoin = !coin; } //if numCompactions odd, flip coin;
else { newCoin = rand.nextBoolean(); } //random coin flip

final FloatBuffer promote = buf.getEvensOrOdds(compactionStart, compactionEnd, coin);
final FloatBuffer promote = buf.getEvensOrOdds(compactionStart, compactionEnd, newCoin);

if (reqDebug != null) {
reqDebug.emitCompactionDetail(compactionStart, compactionEnd, secsToCompact,
promote.getCount(), coin);
promote.getCount(), newCoin);
}

nextBuf.mergeSortIn(promote); //the space was allocated above
coin = newCoin;
buf.trimCount(buf.getCount() - (compactionEnd - compactionStart));
state += 1;
ensureEnoughSections();
cReturn.deltaRetItems = (buf.getCount() - startRetItems) + promote.getCount();
ensureEnoughSections();
cReturn.deltaNomSize = getNomCapacity() - startNomCap;
if (reqDebug != null) { reqDebug.emitCompactionDone(lgWeight); }
return promote;
} //End Compact

/**
Expand Down Expand Up @@ -262,10 +269,10 @@ private boolean ensureEnoughSections() {
&& (sectionSize > MIN_K)
&& ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K))
{
buf.ensureCapacity(2 * NOM_CAP_MULT * (numSections << 1) * ne); //allocate before anything changes
sectionSizeFlt = szf;
sectionSize = ne;
numSections <<= 1;
buf.ensureCapacity(2 * getNomCapacity());
if (reqDebug != null) { reqDebug.emitAdjSecSizeNumSec(lgWeight); }
return true;
}
Expand Down
48 changes: 31 additions & 17 deletions src/main/java/org/apache/datasketches/req/ReqSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@
* not when the sketch is built. Call {@link #ensureCapacity()} right after building the sketch to allocate
* the storage up front instead. Results and the serialized form are the same either way.</p>
*
* <p><b>Exceptions during update.</b> If {@link #update(float)} throws an {@link OutOfMemoryError} before
* the item is stored, the item is not added and the sketch is unchanged. If the error happens later, while
* the sketch compresses, the item has been added and no retained item is lost: the sketch stays valid and
* compresses again on the next update. A merge may have applied part of the other sketch when it throws.</p>
*
* @see QuantilesAPI
*
* @author Edo Liberty
Expand Down Expand Up @@ -469,15 +474,15 @@ public String toString() {
public void update(final float item) {
if (Float.isNaN(item)) { return; }
ensureCapacity(); //first use allocates, before anything changes
final FloatBuffer fbuf = compactors.get(0).getBuffer();
fbuf.append(item); //may grow the buffer; nothing else has changed yet
if (isEmpty()) {
minItem = item;
maxItem = item;
} else {
if (item < minItem) { minItem = item; }
if (item > maxItem) { maxItem = item; }
}
final FloatBuffer fbuf = compactors.get(0).getBuffer();
fbuf.append(item);
retItems++;
totalN++;
if (retItems >= maxNomSize) {
Expand Down Expand Up @@ -554,26 +559,35 @@ private static void checkK(final int k) {
}
}

//Each compaction moves items to the next level only after the space for them has been allocated,
//so a failed allocation never loses items. If an error interrupts compression, the counters are
//recomputed from the compactors, so the sketch stays consistent and the next update compresses again.
private void compress() {
if (reqDebug != null) { reqDebug.emitStartCompress(); }
for (int h = 0; h < compactors.size(); h++) {
final ReqCompactor c = compactors.get(h);
final int compRetItems = c.getBuffer().getCount();
final int compNomCap = c.getNomCapacity();

if (compRetItems >= compNomCap) {
if ((h + 1) >= getNumLevels()) { //at the top?
if (reqDebug != null) { reqDebug.emitMustAddCompactor(); }
grow(); //add a level, increases maxNomSize
try {
for (int h = 0; h < compactors.size(); h++) {
final ReqCompactor c = compactors.get(h);
final int compRetItems = c.getBuffer().getCount();
final int compNomCap = c.getNomCapacity();

if (compRetItems >= compNomCap) {
if ((h + 1) >= getNumLevels()) { //at the top?
if (reqDebug != null) { reqDebug.emitMustAddCompactor(); }
grow(); //add a level, increases maxNomSize
}
c.compact(cReturn, rand, compactors.get(h + 1).getBuffer());
retItems += cReturn.deltaRetItems;
maxNomSize += cReturn.deltaNomSize;
//we specifically decided not to do lazy compression.
}
final FloatBuffer promoted = c.compact(cReturn, rand);
compactors.get(h + 1).getBuffer().mergeSortIn(promoted);
retItems += cReturn.deltaRetItems;
maxNomSize += cReturn.deltaNomSize;
//we specifically decided not to do lazy compression.
}
} catch (final RuntimeException | Error e) {
retItems = computeTotalRetainedItems();
maxNomSize = computeMaxNomSize();
Comment on lines +585 to +586
throw e;
} finally {
reqSV = null;
}
reqSV = null;
if (reqDebug != null) { reqDebug.emitCompressDone(); }
}

Expand Down
116 changes: 116 additions & 0 deletions src/test/java/org/apache/datasketches/req/ReqUpdateFailureTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/


package org.apache.datasketches.req;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;

import java.lang.foreign.MemorySegment;
import java.lang.reflect.Proxy;
import java.util.Random;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator;
import org.testng.annotations.Test;

/**
* Checks that a ReqSketch stays consistent when an error interrupts an update.
*/
public class ReqUpdateFailureTest {
private static final int K = 12;

private static float value(final int i) { return (i * 7919) % 100_003; }

/**
* A compaction allocates the space in the next level before it changes anything. Here the merge into the
* next level fails instead, because that buffer is not sorted. Both compactors must be left as they were.
*/
@Test
public void compactorUnchangedWhenPromotionFails() {
for (final boolean hra : new boolean[] {true, false}) {
final ReqCompactor c = new ReqCompactor((byte) 0, hra, K, null);
final FloatBuffer buf = c.getBuffer();
for (int i = 0; i < c.getNomCapacity(); i++) { buf.append(value(i)); }
buf.sort();
final FloatBuffer next = new FloatBuffer(4, 4, hra);
next.append(2f);
next.append(1f); //not sorted
final byte[] before = c.toByteArray();
final int nextCount = next.getCount();
final boolean coin = c.getCoin();
try {
c.compact(new ReqSketch.CompactorReturn(), new Random(1), next);
fail();
} catch (final SketchesArgumentException e) {
//expected
}
assertEquals(c.toByteArray(), before);
assertEquals(c.getCoin(), coin);
assertEquals(next.getCount(), nextCount);
}
}

/**
* An error during compression leaves the item that started it in the sketch, with counts, bounds and
* min and max that agree with the compactors, and the sketch keeps working.
*/
@Test
public void sketchConsistentWhenCompressionFails() {
for (final boolean hra : new boolean[] {true, false}) {
final int[] calls = {0};
final ReqDebug failOnThirdLevel = (ReqDebug) Proxy.newProxyInstance(ReqDebug.class.getClassLoader(),
new Class<?>[] {ReqDebug.class}, (proxy, method, args) -> {
if (method.getName().equals("emitNewCompactor") && (++calls[0] == 3)) {
throw new IllegalStateException("simulated failure");
}
return null;
});
final ReqSketch sk = ReqSketch.builder().setK(K).setHighRankAccuracy(hra).setReqDebug(failOnThirdLevel).build();
int n = 0;
boolean failed = false;
while (!failed) {
final float item = -n;
try {
sk.update(item);
} catch (final IllegalStateException e) {
failed = true;
}
n++;
assertEquals(sk.getN(), n);
assertEquals(sk.getMinItem(), item); //each item is the new min
checkConsistent(sk);
}
for (int i = 0; i < 10_000; i++) { sk.update(value(i)); }
checkConsistent(sk);
final ReqSketch copy = ReqSketch.heapify(MemorySegment.ofArray(sk.toByteArray()));
assertEquals(copy.toByteArray(), sk.toByteArray());
}
}

private static void checkConsistent(final ReqSketch sk) {
assertEquals(sk.getNumRetained(), sk.computeTotalRetainedItems());
assertEquals(sk.getMaxNomSize(), sk.computeMaxNomSize());
final QuantilesFloatsSketchIterator it = sk.iterator();
long weight = 0;
while (it.next()) { weight += it.getWeight(); }
assertEquals(weight, sk.getN());
}
}
Loading