
# Simple case.

build-remote f1
a#0,SET = foo
b#0,SET = bar
c#0,SET = foobar
----

ingest-external
f1 bounds=(a,cc)
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-cc#inf,RANGEDEL]

iter
first
next
next
next
----
a: (foo, .)
b: (bar, .)
c: (foobar, .)
.

# Above case but with c left out at ingestion time.

reset
----

build-remote f2
a#0,SET = foo
b#0,SET = bar
c#0,SET = foobar
----

ingest-external
f2 bounds=(a,c)
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-c#inf,RANGEDEL]

iter
first
next
next
next
----
a: (foo, .)
b: (bar, .)
.
.

build-remote f3
c#0,SET = foobarbaz
d#0,SET = haha
e#0,SET = something
----

build-remote f4
f#0,SET = foo
g#0,SET = foo
h#0,SET = foo
----

# This ingestion should error out due to the overlap between file spans.

ingest-external
f3 bounds=(c,f)
f4 bounds=(e,h)
----
pebble: external sstables have overlapping ranges

ingest-external
f3 bounds=(c,f)
f4 bounds=(f,hh)
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-c#inf,RANGEDEL]
  000007(000007):[c#11,DELSIZED-f#inf,RANGEDEL]
  000008(000008):[f#12,DELSIZED-hh#inf,RANGEDEL]

iter
first
next
next
next
next
next
next
next
next
----
a: (foo, .)
b: (bar, .)
c: (foobarbaz, .)
d: (haha, .)
e: (something, .)
f: (foo, .)
g: (foo, .)
h: (foo, .)
.

compact a z
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-c#inf,RANGEDEL]
  000007(000007):[c#11,DELSIZED-f#inf,RANGEDEL]
  000008(000008):[f#12,DELSIZED-hh#inf,RANGEDEL]

download a j
----
ok

lsm
----
L6:
  000009:[a#0,SET-b#0,SET]
  000010:[c#0,SET-e#0,SET]
  000011:[f#0,SET-h#0,SET]

iter
first
next
next
next
next
next
next
next
next
----
a: (foo, .)
b: (bar, .)
c: (foobarbaz, .)
d: (haha, .)
e: (something, .)
f: (foo, .)
g: (foo, .)
h: (foo, .)
.

build-remote f5
f#0,SET = foo
g#0,SET = foo
h#0,SET = foo
----

ingest-external
f5 bounds=(ff,fi) synthetic-prefix=f
----

build-remote f6
bf#0,SET = foo
bg#0,SET = foo
bh#0,SET = foo
----

# Test that ingestion with a synthetic prefix or suffix fails on older
# major versions.

reset format-major-version=16
----

build-remote f5
ef#0,SET = foo
eg#0,SET = foo
eh#0,SET = foo
----

ingest-external
f5 bounds=(ff,fi) synthetic-prefix=(f)
----
pebble: format major version too old for synthetic prefix ingestion

ingest-external
f5 bounds=(ff,fi) synthetic-suffix=@5
----
pebble: format major version too old for synthetic suffix ingestion


# Test plumbing for ingestion with suffix replacement
reset
----

build-remote f1
a@1#0,SET = foo
b@2#0,SET = foo
c@1#0,SET = foo
----

build-remote f6
b#0,SET = foo
Span: f-u:{(#0,RANGEDEL)}
----

ingest-external
f1 bounds=(a,d) synthetic-suffix=@5
----

iter
first
next
next
----
a@5: (foo, .)
b@5: (foo, .)
c@5: (foo, .)

# Verify that we require bounds without suffix.
ingest-external
f6 bounds=(a@1,z)
----
pebble: external file bounds start key "a@1" has suffix

ingest-external
f6 bounds=(a,z@10)
----
pebble: external file bounds end key "z@10" has suffix

# Verify that we require valid bounds.
ingest-external
f6 bounds=(c,a)
----
pebble: external file bounds ["c", "a") are invalid

# Test the case when we are ingesting part of a RANGEDEL.
reset
----

batch
set i bar
----

build-remote f6
b#0,SET = foo
Span: f-u:{(#0,RANGEDEL)}
----

ingest-external
f6 bounds=(a,c)
f6 bounds=(g,v)
----

# The previous element cannot be i, because it is inside the [g, v) portion of
# the [f, u) RANGEDEL.
iter
prev
----
b: (foo, .)

# Test the case where the input files need to be reordered during ingest.
reset
----

build-remote f7
a#0,SET = foo
----

build-remote f8
x#0,SET = bar
----

ingest-external
f8 bounds=(x,y)
f7 bounds=(a,b)
----

iter
first
next
----
a: (foo, .)
x: (bar, .)

# Test ViaBackingFileDownload.

reset
----

build-remote f7 block-size=5 index-block-size=5
b#0,SET = ignored
c#0,SET = foo
d#0,SET = haha
e#0,SET = something
x#0,SET = ignored
----

build-remote f8 block-size=100 index-block-size=100
a#0,SET = ignored
g#0,SET = foo
h#0,SET = foo
i#0,SET = foo
z#0,SET = ignored
----

ingest-external
f7 bounds=(gc,gf) synthetic-prefix=g
f8 bounds=(gg,gj) synthetic-prefix=g
----

iter
seek-ge g
next
next
next
next
next
----
gc: (foo, .)
gd: (haha, .)
ge: (something, .)
gg: (foo, .)
gh: (foo, .)
gi: (foo, .)

lsm verbose
----
L6:
  000004(000004):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:1389(1389)
  000005(000005):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:953(953)

download g h via-backing-file-download
----
ok

# observe the size decreases since span copy ignores the unneeded blocks.
lsm verbose
----
L6:
  000006(000006):[gc#10,DELSIZED-gf#inf,RANGEDEL] seqnums:[10-10] points:[gc#10,DELSIZED-gf#inf,RANGEDEL] size:1033(1033)
  000007(000007):[gg#11,DELSIZED-gj#inf,RANGEDEL] seqnums:[11-11] points:[gg#11,DELSIZED-gj#inf,RANGEDEL] size:945(945)

reopen
----

lsm
----
L6:
  000006(000006):[gc#10,DELSIZED-gf#inf,RANGEDEL]
  000007(000007):[gg#11,DELSIZED-gj#inf,RANGEDEL]

iter
seek-ge g
next
next
next
next
next
----
gc: (foo, .)
gd: (haha, .)
ge: (something, .)
gg: (foo, .)
gh: (foo, .)
gi: (foo, .)



# Test seeks with keys outside of the synthetic prefix range.
reset
----

build-remote f9
i#0,SET = foo
----

ingest-external
f9 bounds=(cg,ck) synthetic-prefix=c
----

iter
seek-ge bp
----
ci: (foo, .)

iter
seek-prefix-ge bp
----
.

iter
seek-lt de
----
ci: (foo, .)

# Test compactions with prefix synthesis.
reset
----

build-remote ext
a#0,SET = a
b#0,SET = b
c#0,SET = c
----

ingest-external
ext bounds=(ea,ed) synthetic-prefix=e
----

ingest-external
ext bounds=(ba,bd) synthetic-prefix=b
----

# Write some keys so we actually perform a compaction.
batch
set a a
set c c
set f f
----

compact a z
----

lsm
----
L6:
  000008:[a#0,SET-f#0,SET]

# Make sure we see both ba..bc and ea..ec.
iter
first
next
next
next
next
next
next
next
next
next
----
a: (a, .)
ba: (a, .)
bb: (b, .)
bc: (c, .)
c: (c, .)
ea: (a, .)
eb: (b, .)
ec: (c, .)
f: (f, .)
.

reset
----

build-remote f11
a#0,SET = foo
b#0,SET = bar
----

ingest-external
f11 bounds=(a,c)
----

batch
set a bar
----

iter
first
next
----
a: (bar, .)
b: (bar, .)

replicate 1 2 a z
----
replicated 1 external SSTs

switch 2
----
ok

iter
first
next
----
a: (bar, .)
b: (bar, .)

# Replicate with existing data in L6.

reset
----

build-remote f12
a#0,SET = d1-v1
b#0,SET = d1-v1
c#0,SET = d1-v1
d#0,SET = d-1v1
----

ingest-external
f12 bounds=(a,d)
----

batch
set b d1-v2
----

switch 2
----
ok

batch
set a d2-v1
set b d2-v1
set c d2-v1
set d d2-v1
----

compact a z
----

lsm
----
L6:
  000005:[a#10,SET-d#13,SET]

replicate 1 2 b d
----
replicated 1 external SSTs


iter
first
next
next
next
----
a: (d2-v1, .)
b: (d1-v2, .)
c: (d1-v1, .)
d: (d2-v1, .)

# Replicate with an internal file in L0

reset
----

batch
set d d1-v1
set e d1-v1
----

compact a z
----

lsm
----
L6:
  000005:[d#10,SET-e#11,SET]

build-remote f13
a#0,SET = d1-v1
b#0,SET = d1-v1
----

ingest-external
f13 bounds=(a,c)
----

switch 2
----
ok

batch
set a d2-v1
set b d2-v1
set d d2-v1
set e d2-v1
----

compact a z
----

lsm
----
L6:
  000005:[a#10,SET-e#13,SET]

replicate 1 2 b e
----
replicated 1 external SSTs

iter
first
next
next
next
----
a: (d2-v1, .)
b: (d1-v1, .)
d: (d1-v1, .)
e: (d2-v1, .)

reset
----

build-remote trunctest
a#0,SET = foo
b#0,SET = bar
----

ingest-external
trunctest bounds=(a,c)
----

replicate 1 2 c z
----
replicated 0 external SSTs

reset
----

build-remote inclusive_bounds_test
a#0,SET = foo
b#0,SET = bar
c#0,SET = baz
----

ingest-external
inclusive_bounds_test bounds=(a,c) bounds-are-inclusive=true
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-c#10,DEL]

replicate 1 2 a d
----
replicated 1 external SSTs

iter
first
next
next
----
a: (foo, .)
b: (bar, .)
c: (baz, .)

# Test reuse of external backings.
reset
----

build-remote reuse1
ax#0,SET = ax
ay#0,SET = ay
da#0,SET = da
db#0,SET = db
dc#0,SET = dc
uv#0,SET = uv
----

build-remote reuse2
f#0,SET = f
h#0,SET = h
j#0,SET = j
----

# Test reuse of backings within a single ingestion. We should see only two
# backings; their sizes should be the sum of the corresponding ingestions.
ingest-external
reuse1 bounds=(a,b) size=100
reuse2 bounds=(f,g) size=200
reuse2 bounds=(h,i) size=300
reuse1 bounds=(d,e) size=400
reuse1 bounds=(u,v) size=500
reuse2 bounds=(j,k) size=600
----

lsm verbose
----
L6:
  000004(000004):[a#10,DELSIZED-b#inf,RANGEDEL] seqnums:[10-10] points:[a#10,DELSIZED-b#inf,RANGEDEL] size:100(1000)
  000007(000004):[d#11,DELSIZED-e#inf,RANGEDEL] seqnums:[11-11] points:[d#11,DELSIZED-e#inf,RANGEDEL] size:400(1000)
  000005(000005):[f#12,DELSIZED-g#inf,RANGEDEL] seqnums:[12-12] points:[f#12,DELSIZED-g#inf,RANGEDEL] size:200(1100)
  000006(000005):[h#13,DELSIZED-i#inf,RANGEDEL] seqnums:[13-13] points:[h#13,DELSIZED-i#inf,RANGEDEL] size:300(1100)
  000009(000005):[j#14,DELSIZED-k#inf,RANGEDEL] seqnums:[14-14] points:[j#14,DELSIZED-k#inf,RANGEDEL] size:600(1100)
  000008(000004):[u#15,DELSIZED-v#inf,RANGEDEL] seqnums:[15-15] points:[u#15,DELSIZED-v#inf,RANGEDEL] size:500(1000)

# Test reuse of backings across separate requests.
ingest-external
reuse1 bounds=(xu,xv) synthetic-prefix=x size=5000
reuse2 bounds=(yj,yk) synthetic-prefix=y size=6000
----

# The sizes for the new tables should be capped to the backing size.
lsm verbose
----
L6:
  000004(000004):[a#10,DELSIZED-b#inf,RANGEDEL] seqnums:[10-10] points:[a#10,DELSIZED-b#inf,RANGEDEL] size:100(1000)
  000007(000004):[d#11,DELSIZED-e#inf,RANGEDEL] seqnums:[11-11] points:[d#11,DELSIZED-e#inf,RANGEDEL] size:400(1000)
  000005(000005):[f#12,DELSIZED-g#inf,RANGEDEL] seqnums:[12-12] points:[f#12,DELSIZED-g#inf,RANGEDEL] size:200(1100)
  000006(000005):[h#13,DELSIZED-i#inf,RANGEDEL] seqnums:[13-13] points:[h#13,DELSIZED-i#inf,RANGEDEL] size:300(1100)
  000009(000005):[j#14,DELSIZED-k#inf,RANGEDEL] seqnums:[14-14] points:[j#14,DELSIZED-k#inf,RANGEDEL] size:600(1100)
  000008(000004):[u#15,DELSIZED-v#inf,RANGEDEL] seqnums:[15-15] points:[u#15,DELSIZED-v#inf,RANGEDEL] size:500(1000)
  000010(000004):[xu#16,DELSIZED-xv#inf,RANGEDEL] seqnums:[16-16] points:[xu#16,DELSIZED-xv#inf,RANGEDEL] size:1000(1000)
  000011(000005):[yj#17,DELSIZED-yk#inf,RANGEDEL] seqnums:[17-17] points:[yj#17,DELSIZED-yk#inf,RANGEDEL] size:1100(1100)

batch
del-range a z
----

compact a z
----

ingest-external
reuse1 bounds=(a,b) size=100
reuse2 bounds=(f,g) size=200
----

lsm verbose
----
L6:
  000014(000014):[a#19,DELSIZED-b#inf,RANGEDEL] seqnums:[19-19] points:[a#19,DELSIZED-b#inf,RANGEDEL] size:100(100)
  000015(000015):[f#20,DELSIZED-g#inf,RANGEDEL] seqnums:[20-20] points:[f#20,DELSIZED-g#inf,RANGEDEL] size:200(200)

# Multiple reuse of existing backings in one request.
ingest-external
reuse2 bounds=(h,i) size=300
reuse1 bounds=(d,e) size=400
reuse1 bounds=(u,v) size=500
reuse2 bounds=(j,k) size=600
reuse1 bounds=(xu,xv) synthetic-prefix=x size=700
reuse2 bounds=(yj,yk) synthetic-prefix=y size=800
----

lsm verbose
----
L6:
  000014(000014):[a#19,DELSIZED-b#inf,RANGEDEL] seqnums:[19-19] points:[a#19,DELSIZED-b#inf,RANGEDEL] size:100(100)
  000017(000014):[d#21,DELSIZED-e#inf,RANGEDEL] seqnums:[21-21] points:[d#21,DELSIZED-e#inf,RANGEDEL] size:100(100)
  000015(000015):[f#20,DELSIZED-g#inf,RANGEDEL] seqnums:[20-20] points:[f#20,DELSIZED-g#inf,RANGEDEL] size:200(200)
  000016(000015):[h#22,DELSIZED-i#inf,RANGEDEL] seqnums:[22-22] points:[h#22,DELSIZED-i#inf,RANGEDEL] size:200(200)
  000019(000015):[j#23,DELSIZED-k#inf,RANGEDEL] seqnums:[23-23] points:[j#23,DELSIZED-k#inf,RANGEDEL] size:200(200)
  000018(000014):[u#24,DELSIZED-v#inf,RANGEDEL] seqnums:[24-24] points:[u#24,DELSIZED-v#inf,RANGEDEL] size:100(100)
  000020(000014):[xu#25,DELSIZED-xv#inf,RANGEDEL] seqnums:[25-25] points:[xu#25,DELSIZED-xv#inf,RANGEDEL] size:100(100)
  000021(000015):[yj#26,DELSIZED-yk#inf,RANGEDEL] seqnums:[26-26] points:[yj#26,DELSIZED-yk#inf,RANGEDEL] size:200(200)

# Test that reusing the same backing region with different prefix and suffix
# works as expected. In particular, make sure the synthetic suffix doesn't
# modify cached block data, leading to the next iterator seeing the suffix.

reset
----

build-remote reuse
a@10#0,SET = a
b@11#0,SET = b
----

ingest-external
reuse bounds=(a,c) synthetic-suffix=@20
reuse bounds=(za,zc) synthetic-prefix=z
----

iter
first
next
next
next
----
a@20: (a, .)
b@20: (b, .)
za@10: (a, .)
zb@11: (b, .)

reset
----

build-remote ext3
aa@10#0,SET = a
b@11#0,SET = b
----

ingest-external
ext3 bounds=(a,c)
----

lsm
----
L6:
  000004(000004):[a#10,DELSIZED-c#inf,RANGEDEL]

iter
first
next
next
----
aa@10: (a, .)
b@11: (b, .)
.

build local1
set a foo
----

ingest local1
----

lsm
----
L0.0:
  000005:[a#11,SET-a#11,SET]
L6:
  000004(000004):[a#10,DELSIZED-c#inf,RANGEDEL]

iter
first
next
next
next
----
a: (foo, .)
aa@10: (a, .)
b@11: (b, .)
.

# Test empty external ingestion.
reset
----

build-remote ext
a@10#0,SET = a
b@11#0,SET = b
----

ingest-external
ext bounds=(u,v)
----

lsm
----
L6:
  000004(000004):[u#10,DELSIZED-v#inf,RANGEDEL]

iter
first
----
.

download a z via-backing-file-download
----
ok

# LSM should be empty now.
lsm
----

iter
first
----
.

# Test with two external files, one empty.
ingest-external
ext bounds=(a,c)
----

ingest-external
ext bounds=(u,v)
----

lsm
----
L6:
  000006(000006):[a#11,DELSIZED-c#inf,RANGEDEL]
  000007(000006):[u#12,DELSIZED-v#inf,RANGEDEL]

iter
first
next
next
----
a@10: (a, .)
b@11: (b, .)
.

download a z
----
ok

lsm
----
L6:
  000008:[a@10#0,SET-b@11#0,SET]

iter
first
next
next
----
a@10: (a, .)
b@11: (b, .)
.


# Tests with range tombstones.

reset
----

build-remote points.sst
a@1#0,SET = va
b@1#0,SET = vb
c@1#0,SET = vc
d@1#0,SET = vc
----

build-remote ranges.sst
Span: b-b1:{(#0,RANGEDEL)}
----

ingest-external
points.sst bounds=(a,d)
----

ingest-external
ranges.sst bounds=(a,d)
----

iter
first
next
next
next
----
a@1: (va, .)
c@1: (vc, .)
.
.

ingest-external
points.sst bounds=(p1_a,p1_d) synthetic-prefix=p1_
points.sst bounds=(p2_b,p2_c) synthetic-prefix=p2_
----

ingest-external
ranges.sst bounds=(p1_a,p1_d) synthetic-prefix=p1_
ranges.sst bounds=(p3_a,p3_d) synthetic-prefix=p3_
----

iter
first
next
next
next
next
next
----
a@1: (va, .)
c@1: (vc, .)
p1_a@1: (va, .)
p1_c@1: (vc, .)
p2_b@1: (vb, .)
.

iter
seek-ge p
next
next
next
----
p1_a@1: (va, .)
p1_c@1: (vc, .)
p2_b@1: (vb, .)
.

iter
seek-lt p3
prev
prev
prev
prev
prev
----
p2_b@1: (vb, .)
p1_c@1: (vc, .)
p1_a@1: (va, .)
c@1: (vc, .)
a@1: (va, .)
.

# Tests with range keys.

reset
----

build-remote points-and-ranges.sst
a@1#0,SET = va
b@1#0,SET = vb
c@1#0,SET = vc
d@1#0,SET = vc
Span: b-c1:{(#0,RANGEKEYSET,@2,vrange)}
----

ingest-external
points-and-ranges.sst bounds=(a,e) has-range-keys
----

iter
first
next
next
next
next
next
----
a@1: (va, .)
b: (., [b-"c1") @2=vrange UPDATED)
b@1: (vb, [b-"c1") @2=vrange)
c@1: (vc, [b-"c1") @2=vrange)
d@1: (vc, . UPDATED)
.

ingest-external
points-and-ranges.sst synthetic-prefix=x_ bounds=(x_a,x_d) has-range-keys
----

iter
last
prev
prev
prev
prev
prev
----
x_c@1: (vc, [x_b-"x_c1") @2=vrange UPDATED)
x_b@1: (vb, [x_b-"x_c1") @2=vrange)
x_b: (., [x_b-"x_c1") @2=vrange)
x_a@1: (va, . UPDATED)
d@1: (vc, .)
c@1: (vc, [b-"c1") @2=vrange UPDATED)

# Test with loose end bound and no point keys at/after the last range end kye.
ingest-external
points-and-ranges.sst synthetic-prefix=z_ bounds=(z_a,z_c2) has-range-keys
----

iter
seek-ge y
next
next
next
next
----
z_a@1: (va, .)
z_b: (., [z_b-"z_c1") @2=vrange UPDATED)
z_b@1: (vb, [z_b-"z_c1") @2=vrange)
z_c@1: (vc, [z_b-"z_c1") @2=vrange)
.

ingest-external
points-and-ranges.sst synthetic-prefix=y_ synthetic-suffix=@4 bounds=(y_a,y_d) has-range-keys
----

iter
seek-lt y
next
next
next
next
next
----
x_c@1: (vc, [x_b-"x_c1") @2=vrange UPDATED)
y_a@4: (va, . UPDATED)
y_b: (., [y_b-"y_c1") @4=vrange UPDATED)
y_b@4: (vb, [y_b-"y_c1") @4=vrange)
y_c@4: (vc, [y_b-"y_c1") @4=vrange)
z_a@1: (va, . UPDATED)
