Skip to content

Implemented iterator for variable length structs. #164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
badrishc merged 16 commits into from
Aug 27, 2019
Merged

Implemented iterator for variable length structs. #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ba19835
Implemented iterator for variable length structs.
Aug 9, 2019
4c9de49
Return VariableLengthBlittableScanIterator from VariableLengthBlittab…
Aug 10, 2019
49ee7d6
Added capacity to record when using variable length struct allocator.
Aug 16, 2019
f4b8a9a
Call AllocatorBase.WriteInfo instead of directly invoking RecordInfo.…
Aug 16, 2019
3b65d50
Return estimated record size if stored capacity is 0.
Aug 16, 2019
30b3a14
Fixed typo that was causing failures in tests.
Aug 16, 2019
d78f550
Merge pull request #1 from microsoft/master
marius-klimantavicius Aug 19, 2019
151ba13
Merge branch 'master' into varlen_scan_iterator
Aug 19, 2019
76b1be2
Merge branch 'master' into varlen_scan_iterator
marius-klimantavicius Aug 19, 2019
5490842
ConcurrentWriter/InPlaceUpdater returns bool to indicate whether it i…
marius-klimantavicius Aug 20, 2019
53497c9
Merge branch 'varlen_scan_iterator' of https://github.com/marius-klim…
Aug 20, 2019
34cfbd4
Modified IFunctions.ConcurrentWriter and IFunctions.InPlaceUpdater to…
Aug 20, 2019
1dee257
Added Delete to IFasterKV.
Aug 23, 2019
2594ccc
Do not use C# 7.3 features
marius-klimantavicius Aug 23, 2019
5b3f350
Variable length/blittable iterator - load record info from frame or h…
marius-klimantavicius Aug 23, 2019
a28ef2b
Merge branch 'master' into varlen_scan_iterator
badrishc Aug 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cs/benchmark/Functions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ public void SingleWriter(ref Key key, ref Value src, ref Value dst)
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ConcurrentWriter(ref Key key, ref Value src, ref Value dst)
public bool ConcurrentWriter(ref Key key, ref Value src, ref Value dst)
{
dst = src;
return true;
}

// RMW functions
Expand All @@ -67,9 +68,10 @@ public void InitialUpdater(ref Key key, ref Input input, ref Value value)
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void InPlaceUpdater(ref Key key, ref Input input, ref Value value)
public bool InPlaceUpdater(ref Key key, ref Input input, ref Value value)
{
value.value += input.value;
return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down
5 changes: 3 additions & 2 deletions cs/playground/ClassCache/Types.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,10 @@ public void ConcurrentReader(ref CacheKey key, ref CacheInput input, ref CacheVa
dst.value = value;
}

public void ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
public bool ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
{
dst = src;
return true;
}

public void CopyUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue oldValue, ref CacheValue newValue)
Expand All @@ -107,7 +108,7 @@ public void InitialUpdater(ref CacheKey key, ref CacheInput input, ref CacheValu
throw new NotImplementedException();
}

public void InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
public bool InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
{
throw new NotImplementedException();
}
Expand Down
5 changes: 3 additions & 2 deletions cs/playground/ClassCacheMT/Types.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,10 @@ public void ConcurrentReader(ref CacheKey key, ref CacheInput input, ref CacheVa
dst.value = value;
}

public void ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
public bool ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
{
dst = src;
return true;
}

public void CopyUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue oldValue, ref CacheValue newValue)
Expand All @@ -107,7 +108,7 @@ public void InitialUpdater(ref CacheKey key, ref CacheInput input, ref CacheValu
throw new NotImplementedException();
}

public void InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
public bool InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
{
throw new NotImplementedException();
}
Expand Down
4 changes: 2 additions & 2 deletions cs/playground/ClassSample/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ public class MyFunctions : IFunctions<MyKey, MyValue, MyInput, MyOutput, MyConte
{
public void InitialUpdater(ref MyKey key, ref MyInput input, ref MyValue value) => value.value = input.value;
public void CopyUpdater(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue) => newValue = oldValue;
public void InPlaceUpdater(ref MyKey key, ref MyInput input, ref MyValue value) => value.value += input.value;
public bool InPlaceUpdater(ref MyKey key, ref MyInput input, ref MyValue value) { value.value += input.value; return true; }


public void SingleReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst) => dst.value = value;
public void SingleWriter(ref MyKey key, ref MyValue src, ref MyValue dst) => dst = src;
public void ConcurrentReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst) => dst.value = value;
public void ConcurrentWriter(ref MyKey key, ref MyValue src, ref MyValue dst) => dst = src;
public bool ConcurrentWriter(ref MyKey key, ref MyValue src, ref MyValue dst) { dst = src; return true; }

public void ReadCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, MyContext ctx, Status status) { }
public void UpsertCompletionCallback(ref MyKey key, ref MyValue value, MyContext ctx) { }
Expand Down
6 changes: 4 additions & 2 deletions cs/playground/FixedLenStructSample/Functions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ public void ConcurrentReader(ref FixedLenKey key, ref string input, ref FixedLen
dst = value.ToString();
}

public void ConcurrentWriter(ref FixedLenKey key, ref FixedLenValue src, ref FixedLenValue dst)
public bool ConcurrentWriter(ref FixedLenKey key, ref FixedLenValue src, ref FixedLenValue dst)
{
src.CopyTo(ref dst);
return true;
}

public void CopyUpdater(ref FixedLenKey key, ref string input, ref FixedLenValue oldValue, ref FixedLenValue newValue)
Expand All @@ -37,8 +38,9 @@ public void InitialUpdater(ref FixedLenKey key, ref string input, ref FixedLenVa
{
}

public void InPlaceUpdater(ref FixedLenKey key, ref string input, ref FixedLenValue value)
public bool InPlaceUpdater(ref FixedLenKey key, ref string input, ref FixedLenValue value)
{
return true;
}

public void ReadCompletionCallback(ref FixedLenKey key, ref string input, ref string output, Empty ctx, Status status)
Expand Down
5 changes: 3 additions & 2 deletions cs/playground/PeriodicCompaction/Types.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,10 @@ public void ConcurrentReader(ref CacheKey key, ref CacheInput input, ref CacheVa
dst.value = value;
}

public void ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
public bool ConcurrentWriter(ref CacheKey key, ref CacheValue src, ref CacheValue dst)
{
dst = src;
return true;
}

public void CopyUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue oldValue, ref CacheValue newValue)
Expand All @@ -107,7 +108,7 @@ public void InitialUpdater(ref CacheKey key, ref CacheInput input, ref CacheValu
throw new NotImplementedException();
}

public void InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
public bool InPlaceUpdater(ref CacheKey key, ref CacheInput input, ref CacheValue value)
{
throw new NotImplementedException();
}
Expand Down
9 changes: 5 additions & 4 deletions cs/playground/StructSample/Functions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ public class Sample1Funcs : IFunctions<long, long, long, long, Empty>

// Write functions
public void SingleWriter(ref long key, ref long src, ref long dst) => dst = src;
public void ConcurrentWriter(ref long key, ref long src, ref long dst) => dst = src;
public bool ConcurrentWriter(ref long key, ref long src, ref long dst) { dst = src; return true; }

// RMW functions
public void InitialUpdater(ref long key, ref long input, ref long value) => value = input;
public void CopyUpdater(ref long key, ref long input, ref long oldv, ref long newv) => newv = oldv + input;
public void InPlaceUpdater(ref long key, ref long input, ref long value) => value += input;
public bool InPlaceUpdater(ref long key, ref long input, ref long value) { value += input; return true; }

// Completion callbacks
public void ReadCompletionCallback(ref long key, ref long input, ref long output, Empty ctx, Status s) { }
Expand All @@ -46,7 +46,7 @@ public class Sample2Funcs : IFunctions<Key, Value, Input, Output, Empty>

// Write functions
public void SingleWriter(ref Key key, ref Value src, ref Value dst) => dst = src;
public void ConcurrentWriter(ref Key key, ref Value src, ref Value dst) => dst = src;
public bool ConcurrentWriter(ref Key key, ref Value src, ref Value dst) { dst = src; return true; }

// RMW functions
public void InitialUpdater(ref Key key, ref Input input, ref Value value)
Expand All @@ -59,10 +59,11 @@ public void CopyUpdater(ref Key key, ref Input input, ref Value oldValue, ref Va
newValue.vfield1 = oldValue.vfield1 + input.ifield1;
newValue.vfield2 = oldValue.vfield2 + input.ifield2;
}
public void InPlaceUpdater(ref Key key, ref Input input, ref Value value)
public bool InPlaceUpdater(ref Key key, ref Input input, ref Value value)
{
value.vfield1 += input.ifield1;
value.vfield2 += input.ifield2;
return true;
}

// Completion callbacks
Expand Down
9 changes: 5 additions & 4 deletions cs/playground/StructSampleCore/Functions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ public class Sample1Funcs : IFunctions<long, long, long, long, Empty>

// Write functions
public void SingleWriter(ref long key, ref long src, ref long dst) => dst = src;
public void ConcurrentWriter(ref long key, ref long src, ref long dst) => dst = src;
public bool ConcurrentWriter(ref long key, ref long src, ref long dst) { dst = src; return true; }

// RMW functions
public void InitialUpdater(ref long key, ref long input, ref long value) => value = input;
public void CopyUpdater(ref long key, ref long input, ref long oldv, ref long newv) => newv = oldv + input;
public void InPlaceUpdater(ref long key, ref long input, ref long value) => value += input;
public bool InPlaceUpdater(ref long key, ref long input, ref long value) { value += input; return true; }

// Completion callbacks
public void ReadCompletionCallback(ref long key, ref long input, ref long output, Empty ctx, Status s) { }
Expand All @@ -46,7 +46,7 @@ public class Sample2Funcs : IFunctions<Key, Value, Input, Output, Empty>

// Write functions
public void SingleWriter(ref Key key, ref Value src, ref Value dst) => dst = src;
public void ConcurrentWriter(ref Key key, ref Value src, ref Value dst) => dst = src;
public bool ConcurrentWriter(ref Key key, ref Value src, ref Value dst) { dst = src; return true; }

// RMW functions
public void InitialUpdater(ref Key key, ref Input input, ref Value value)
Expand All @@ -59,10 +59,11 @@ public void CopyUpdater(ref Key key, ref Input input, ref Value oldValue, ref Va
newValue.vfield1 = oldValue.vfield1 + input.ifield1;
newValue.vfield2 = oldValue.vfield2 + input.ifield2;
}
public void InPlaceUpdater(ref Key key, ref Input input, ref Value value)
public bool InPlaceUpdater(ref Key key, ref Input input, ref Value value)
{
value.vfield1 += input.ifield1;
value.vfield2 += input.ifield2;
return true;
}

// Completion callbacks
Expand Down
6 changes: 4 additions & 2 deletions cs/playground/SumStore/SumStoreTypes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,10 @@ public void SingleWriter(ref AdId key, ref NumClicks src, ref NumClicks dst)
dst = src;
}

public void ConcurrentWriter(ref AdId key, ref NumClicks src, ref NumClicks dst)
public bool ConcurrentWriter(ref AdId key, ref NumClicks src, ref NumClicks dst)
{
dst = src;
return true;
}

// RMW functions
Expand All @@ -95,9 +96,10 @@ public void InitialUpdater(ref AdId key, ref Input input, ref NumClicks value)
value = input.numClicks;
}

public void InPlaceUpdater(ref AdId key, ref Input input, ref NumClicks value)
public bool InPlaceUpdater(ref AdId key, ref Input input, ref NumClicks value)
{
Interlocked.Add(ref value.numClicks, input.numClicks.numClicks);
return true;
}

public void CopyUpdater(ref AdId key, ref Input input, ref NumClicks oldValue, ref NumClicks newValue)
Expand Down
6 changes: 4 additions & 2 deletions cs/playground/VarLenStructSample/Functions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,20 @@ public void SingleWriter(ref VarLenType key, ref VarLenType src, ref VarLenType
src.CopyTo(ref dst);
}

public void ConcurrentWriter(ref VarLenType key, ref VarLenType src, ref VarLenType dst)
public bool ConcurrentWriter(ref VarLenType key, ref VarLenType src, ref VarLenType dst)
{
src.CopyTo(ref dst);
return true;
}

// RMW functions
public void InitialUpdater(ref VarLenType key, ref int[] input, ref VarLenType value)
{
}

public void InPlaceUpdater(ref VarLenType key, ref int[] input, ref VarLenType value)
public bool InPlaceUpdater(ref VarLenType key, ref int[] input, ref VarLenType value)
{
return true;
}

public void CopyUpdater(ref VarLenType key, ref int[] input, ref VarLenType oldValue, ref VarLenType newValue)
Expand Down
90 changes: 56 additions & 34 deletions cs/src/core/Allocator/BlittableScanIterator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public class BlittableScanIterator<Key, Value> : IFasterScanIterator<Key, Value>

private bool first = true;
private long currentAddress, nextAddress;
private long currentPhysicalAddress;

/// <summary>
/// Current address
Expand Down Expand Up @@ -72,17 +73,31 @@ public unsafe BlittableScanIterator(BlittableAllocator<Key, Value> hlog, long be
}

/// <summary>
/// Get next record in iterator
/// Gets reference to current key
/// </summary>
/// <param name="recordInfo"></param>
/// <param name="key"></param>
/// <param name="value"></param>
/// <returns></returns>
public bool GetNext(out RecordInfo recordInfo, out Key key, out Value value)
public ref Key GetKey()
{
return ref hlog.GetKey(currentPhysicalAddress);
}

/// <summary>
/// Gets reference to current value
/// </summary>
/// <returns></returns>
public ref Value GetValue()
{
return ref hlog.GetValue(currentPhysicalAddress);
}

/// <summary>
/// Get next record
/// </summary>
/// <param name="recordInfo"></param>
/// <returns>True if record found, false if end of scan</returns>
public bool GetNext(out RecordInfo recordInfo)
{
recordInfo = default(RecordInfo);
key = default(Key);
value = default(Value);

currentAddress = nextAddress;
while (true)
Expand All @@ -109,47 +124,54 @@ public bool GetNext(out RecordInfo recordInfo, out Key key, out Value value)
if (currentAddress < hlog.HeadAddress)
BufferAndLoad(currentAddress, currentPage, currentPage % frameSize);

var recordSize = hlog.GetRecordSize(hlog.GetPhysicalAddress(currentAddress));
var physicalAddress = default(long);
if (currentAddress >= hlog.HeadAddress)
physicalAddress = hlog.GetPhysicalAddress(currentAddress);
else
physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);

// Check if record fits on page, if not skip to next page
var recordSize = hlog.GetRecordSize(physicalAddress);
if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize)
{
currentAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits;
continue;
}


if (currentAddress >= hlog.HeadAddress)
ref var info = ref hlog.GetInfo(physicalAddress);
if (info.Invalid || info.IsNull())
{
// Read record from cached page memory
var _physicalAddress = hlog.GetPhysicalAddress(currentAddress);

if (hlog.GetInfo(_physicalAddress).Invalid)
{
currentAddress += hlog.GetRecordSize(_physicalAddress);
continue;
}

recordInfo = hlog.GetInfo(_physicalAddress);
key = hlog.GetKey(_physicalAddress);
value = hlog.GetValue(_physicalAddress);
nextAddress = currentAddress + hlog.GetRecordSize(_physicalAddress);
return true;
currentAddress += recordSize;
continue;
}

var physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset);
currentPhysicalAddress = physicalAddress;
recordInfo = info;
nextAddress = currentAddress + recordSize;
return true;
}
}

if (hlog.GetInfo(physicalAddress).Invalid)
{
currentAddress += hlog.GetRecordSize(physicalAddress);
continue;
}
/// <summary>
/// Get next record in iterator
/// </summary>
/// <param name="recordInfo"></param>
/// <param name="key"></param>
/// <param name="value"></param>
/// <returns></returns>
public bool GetNext(out RecordInfo recordInfo, out Key key, out Value value)
{
key = default(Key);
value = default(Value);

recordInfo = hlog.GetInfo(physicalAddress);
key = hlog.GetKey(physicalAddress);
value = hlog.GetValue(physicalAddress);
nextAddress = currentAddress + hlog.GetRecordSize(physicalAddress);
if (GetNext(out recordInfo))
{
key = GetKey();
value = GetValue();
return true;
}

return false;
}

/// <summary>
Expand Down
Loading