Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions engine/src/main/java/com/arcadedb/database/Binary.java
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,12 @@ public int putBytes(final byte[] value, final int size) {
return bytesUsed + size;
}

@Override
public int putBytes(final int index, final byte[] value, final int size) {
position(index);
return putBytes(value, size);
}

@Override
public void putByteArray(final int index, final byte[] value) {
position(index);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ public interface BinaryStructure {

int putBytes(byte[] value, int size);

int putBytes(int index, byte[] value, int size);

void putByteArray(int index, byte[] value);

void putByteArray(int index, byte[] value, int offset, int length);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -579,12 +579,12 @@ public TransactionPhase1 commit1stPhase(final boolean isLeader) {
for (MutablePage page : modifiedPages.values()) {
final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false);
if (bucket != null)
bucket.compressPage(page);
bucket.compressPage(page, false);
}
for (MutablePage page : newPages.values()) {
final LocalBucket bucket = localSchema.getBucketById(page.getPageId().getFileId(), false);
if (bucket != null)
bucket.compressPage(page);
bucket.compressPage(page, false);
}

for (final Iterator<MutablePage> it = modifiedPages.values().iterator(); it.hasNext(); ) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public void compress() {
final MutablePage page = database.getTransaction()
.getPageToModify(new PageId(database, bucket.getFileId(), i), bucket.getPageSize(), false);

bucket.compressPage(page);
bucket.compressPage(page, true);

++pageBatch;

Expand Down
46 changes: 33 additions & 13 deletions engine/src/main/java/com/arcadedb/engine/LocalBucket.java
Original file line number Diff line number Diff line change
Expand Up @@ -910,8 +910,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder

MutablePage page = null;
try {
page = database.getTransaction()
.getPageToModify(new PageId(database, file.getFileId(), pageId), pageSize, false);
page = database.getTransaction().getPageToModify(new PageId(database, file.getFileId(), pageId), pageSize, false);

final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET);
if (positionInPage >= recordCountInPage)
Expand Down Expand Up @@ -983,6 +982,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder
page.writeZeros(recordPositionInPage + 1, (int) (recordSize[0] + recordSize[1] - 1));
} catch (Exception e) {
// IGNORE IT
LogManager.instance().log(this, Level.SEVERE, "Error on wiping out page content", e);
}
}

Expand All @@ -993,6 +993,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder
if (reuseSpaceMode.ordinal() > REUSE_SPACE_MODE.MEDIUM.ordinal()) {
// UPDATE THE STATISTICS
final PageAnalysis pageAnalysis = new PageAnalysis(page);
pageAnalysis.totalRecordsInPage = recordCountInPage;
getFreeSpaceInPage(pageAnalysis);
updatePageStatistics(pageId, pageAnalysis.spaceAvailableInCurrentPage, (int) ((recordSize[0] + recordSize[1]) * -1));
} else
Expand Down Expand Up @@ -1020,7 +1021,7 @@ private void deleteRecordInternal(final RID rid, final boolean deletePlaceholder
}
}

public void compressPage(final MutablePage page) throws IOException {
public void compressPage(final MutablePage page, final boolean forceWipeOut) throws IOException {
final short recordCountInPage = page.readShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET);

final List<int[]> orderedRecordContentInPage = getOrderedRecordsInPage(page, recordCountInPage, false);
Expand All @@ -1030,6 +1031,7 @@ public void compressPage(final MutablePage page) throws IOException {
// RESET RECORD COUNTER TO 0
page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) 0);
LogManager.instance().log(this, Level.FINE, "Update record count from %d to 0 in page %s", recordCountInPage, page.pageId);
wipeOutFreeSpace(page, (short) 0);
}
return;
}
Expand Down Expand Up @@ -1059,6 +1061,27 @@ public void compressPage(final MutablePage page) throws IOException {
.log(this, Level.FINE, "Update record count from %d to %d in page %s", recordCountInPage, newRecordCount, page.pageId);
page.writeShort(PAGE_RECORD_COUNT_IN_PAGE_OFFSET, (short) newRecordCount);
}

}

if (forceWipeOut || !holes.isEmpty())
wipeOutFreeSpace(page, recordCountInPage);
}

private void wipeOutFreeSpace(final MutablePage page, final short recordCountInPage) throws IOException {
if (database.getConfiguration().getValueAsBoolean(GlobalConfiguration.BUCKET_WIPEOUT_ONDELETE)) {
// WIPE OUT FREE SPACE IN THE PAGE. THIS HELPS WITH THE BACKUP OF DATABASE INCREASING THE COMPRESSION RATE
try {
final PageAnalysis pageAnalysis = new PageAnalysis(page);
pageAnalysis.totalRecordsInPage = recordCountInPage;
getFreeSpaceInPage(pageAnalysis);

if (pageAnalysis.spaceAvailableInCurrentPage > 0)
page.writeZeros(pageAnalysis.newRecordPositionInPage, page.getMaxContentSize() - pageAnalysis.newRecordPositionInPage);
} catch (Exception e) {
// IGNORE IT
LogManager.instance().log(this, Level.SEVERE, "Error on wiping out page content", e);
}
}
}

Expand Down Expand Up @@ -1171,9 +1194,8 @@ else if (recordSize[0] == FIRST_CHUNK || recordSize[0] == NEXT_CHUNK) {
continue;
}
} catch (Exception e) {
LogManager.instance()
.log(this, Level.SEVERE, "Error on loading record #" + fileId + ":" + (page.pageId.getPageNumber() * maxRecordsInPage)
+ positionInPage);
LogManager.instance().log(this, Level.SEVERE,
"Error on loading record #" + fileId + ":" + (page.pageId.getPageNumber() * maxRecordsInPage) + positionInPage);
continue;
}

Expand Down Expand Up @@ -1220,8 +1242,8 @@ private Binary loadMultiPageRecord(final RID originalRID, BasePage page, int rec
// AVOID INFINITE LOOP?
LogManager.instance().log(this, Level.SEVERE,
"Infinite loop on loading multi-page record " + originalRID + " chunk " + chunkPageId + "/" + chunkPositionInPage);
throw new DatabaseOperationException("Infinite loop on loading multi-page record " + originalRID + " chunk "
+ chunkPageId + "/" + chunkPositionInPage);
throw new DatabaseOperationException(
"Infinite loop on loading multi-page record " + originalRID + " chunk " + chunkPageId + "/" + chunkPositionInPage);
}

page = nextPage;
Expand Down Expand Up @@ -1280,8 +1302,7 @@ private void writeMultiPageRecord(final RID originalRID, final Binary buffer, Mu

final int spaceNeededForChunk = bufferSize + 2 + INT_SERIALIZED_SIZE + LONG_SERIALIZED_SIZE;

final PageAnalysis pageAnalysis = findAvailableSpace(currentPage.pageId.getPageNumber(), spaceNeededForChunk,
txPageCounter);
final PageAnalysis pageAnalysis = findAvailableSpace(currentPage.pageId.getPageNumber(), spaceNeededForChunk, txPageCounter);
if (!pageAnalysis.createNewPage) {
nextPage = database.getTransaction().getPageToModify(pageAnalysis.page.pageId, pageSize, false);
newPosition = pageAnalysis.newRecordPositionInPage;
Expand Down Expand Up @@ -1682,9 +1703,8 @@ private PageAnalysis findAvailableSpaceFromStatistics(final int currentPageId, f
* 3. if the tree map is full (size > MAX_PAGES_GATHER_STATS), stop
*/
public void gatherPageStatistics() {
if (timeOfLastStats == 0L ||
(System.currentTimeMillis() - timeOfLastStats > MAX_TIMEOUT_GATHER_STATS && changesFromLastStats > 0)
)
if (timeOfLastStats == 0L || (System.currentTimeMillis() - timeOfLastStats > MAX_TIMEOUT_GATHER_STATS
&& changesFromLastStats > 0))
try {
int txPageCount = getTotalPages();

Expand Down
25 changes: 18 additions & 7 deletions engine/src/main/java/com/arcadedb/engine/MutablePage.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@
*/
package com.arcadedb.engine;

import com.arcadedb.GlobalConfiguration;
import com.arcadedb.database.Binary;
import com.arcadedb.database.DatabaseFactory;
import com.arcadedb.database.TrackableBinary;
import com.arcadedb.log.LogManager;

import java.nio.*;
import java.util.*;
import java.util.logging.*;

/**
* Mutable page that accepts updates. It keeps track of the modified bytes.
Expand All @@ -33,17 +36,21 @@
* @author Luca Garulli (l.garulli@arcadedata.com)
*/
public class MutablePage extends BasePage implements TrackableContent {
private int modifiedRangeFrom = Integer.MAX_VALUE;
private int modifiedRangeTo = -1;
private WALFile walFile;
private static final byte[] ZERO_BYTES_ARRAY;
private int modifiedRangeFrom = Integer.MAX_VALUE;
private int modifiedRangeTo = -1;
private WALFile walFile;

static {
ZERO_BYTES_ARRAY = new byte[GlobalConfiguration.BUCKET_DEFAULT_PAGE_SIZE.getValueAsInteger()];
}

public MutablePage(final PageId pageId, final int size) {
this(pageId, size, new byte[size], 0, 0);
updateModifiedRange(0, size - 1);
}

public MutablePage(final PageId pageId, final int size, final byte[] array, final int version,
final int contentSize) {
public MutablePage(final PageId pageId, final int size, final byte[] array, final int version, final int contentSize) {
super(pageId, size, array, version, contentSize);
}

Expand Down Expand Up @@ -150,8 +157,12 @@ public void writeByteArray(int index, final byte[] content, final int contentOff
public void writeZeros(int index, final int contentLength) {
index += PAGE_HEADER_SIZE;
checkBoundariesOnWrite(index, contentLength);
for (int i = 0; i < contentLength; i++)
this.content.putByte(index + i, (byte) 0);
if (contentLength <= ZERO_BYTES_ARRAY.length)
// FAST COPY
this.content.putByteArray(index, ZERO_BYTES_ARRAY, 0, contentLength);
else
for (int i = 0; i < contentLength; i++)
this.content.putByte(index + i, (byte) 0);
}

public int writeString(final int index, final String content) {
Expand Down