mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
9f271467fb
We add endpoints and protobuf encode/decode to allow for sending sharded requests over the wire in protobuf, so we can take our sharded data and send it to other nodes if needed. This is a squash of >15 other commits, so a bit of history is relevant: The Request type had FieldTypes in it because the field type information was needed for sharding because sorting requires that information. We change this around to make the external sharding operation require the field types, and curry that through the codec -- the codec is needed to tell the request how it shards. (This is because the correct sorting order varies by field type.) Requests (and ShardedRequests) no longer have that table in them. And then we hit a nasty bug in production and RCA showed that our testing wasn't good enough and we need to be more careful, and I discovered that test coverage in this package was around 70%. So, the other big thing here is coverage testing; in order to make coverage testing viable and programmatically testable, we have added the ability to render requests *back* to JSON. This is not a great idea, but it does allow us to do a lot of sanity-checking and verify that the encodings we're using are consistent and correct. This, plus some specific tests of decoding specific flawed inputs, has caught a number of issues. Which are now fixed! A lot of internal API surface got slightly changed, in ways that make it simpler to work with. For instance, the (*FieldOperation).TranslateUnsigned function doesn't really need to exist; we can just have a non-method translate function for unsigned and for signed, and use them based on field type. The stable translation hack used for testing had a bug that could allow it to end up producing incorrect results if you asked it to translate an ID first rather than exclusively asking it to translate strings first, this has been corrected. (This is a bug fix in code that was added partway through creating this, but is tricky enough to mention its own comment.) Test coverage is now just over 90%, and a lot of what's left is error-check returns that may well be actually unreachable unless, say, the documentation for encoding/json is full of lies. Which it probably is.
114 lines
3 KiB
Go
114 lines
3 KiB
Go
// Copyright 2021 Molecula Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ingest
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
)
|
|
|
|
type badTranslator struct{}
|
|
|
|
func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
|
|
if len(keys) == 0 {
|
|
return nil, errors.New("no keys")
|
|
}
|
|
m := make(map[string]uint64)
|
|
skip := true
|
|
for i, k := range keys {
|
|
if skip {
|
|
skip = false
|
|
continue
|
|
}
|
|
m[k] = uint64(i)
|
|
}
|
|
out := make([]uint64, len(keys)-1)
|
|
for i := range out {
|
|
out[i] = uint64(i)
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func TestStringTableErrors(t *testing.T) {
|
|
tbl := NewStringTable()
|
|
btr := badTranslator{}
|
|
_, keyErr := tbl.MakeIDMap(btr)
|
|
if keyErr == nil {
|
|
t.Fatalf("expected error passed up from failed translate, didn't get it")
|
|
}
|
|
a1, err := tbl.ID([]byte("a"))
|
|
if err != nil {
|
|
t.Fatalf("getting translation for key: %v", err)
|
|
}
|
|
b1, err := tbl.ID([]byte("b"))
|
|
if err != nil {
|
|
t.Fatalf("getting translation for key: %v", err)
|
|
}
|
|
_, keyErr = tbl.MakeIDMap(btr)
|
|
if keyErr == nil {
|
|
t.Fatalf("expected error for short translate, didn't get it")
|
|
}
|
|
tr := newStableTranslator()
|
|
_, err = tr.TranslateKeys("c", "d")
|
|
if err != nil {
|
|
t.Fatalf("translating stray keys: %v", err)
|
|
}
|
|
m, err := tbl.MakeIDMap(tr)
|
|
if err != nil {
|
|
t.Fatalf("creating lookup: %v", err)
|
|
}
|
|
var y = []uint64{a1, b1}
|
|
err = translateUnsigned(m, y)
|
|
if err != nil {
|
|
t.Fatalf("unexpected unsigned translation error: %v", err)
|
|
}
|
|
trResults, err := tr.TranslateKeys("a", "b")
|
|
if err != nil {
|
|
t.Fatalf("unexpected translation error: %v", err)
|
|
}
|
|
if y[0] != trResults["a"] {
|
|
t.Fatalf("expected %d, got %d", trResults["a"], y[0])
|
|
}
|
|
if y[1] != trResults["b"] {
|
|
t.Fatalf("expected %d, got %d", trResults["b"], y[1])
|
|
}
|
|
y[0] = a1
|
|
y[1] = (a1 + b1 + 1) // assumed not to be any of them
|
|
err = translateUnsigned(m, y)
|
|
if err == nil {
|
|
t.Fatalf("no error from translating invalid table")
|
|
}
|
|
z := []int64{int64(a1), int64(b1)}
|
|
err = translateSigned(m, z)
|
|
if err != nil {
|
|
t.Fatalf("unexpected unsigned translation error: %v", err)
|
|
}
|
|
if uint64(z[0]) != trResults["a"] {
|
|
t.Fatalf("expected %d, got %d", trResults["a"], z[0])
|
|
}
|
|
if uint64(z[1]) != trResults["b"] {
|
|
t.Fatalf("expected %d, got %d", trResults["b"], z[1])
|
|
}
|
|
z[0] = int64(a1)
|
|
z[1] = int64(a1 + b1 + 1) // assumed not to be any of them
|
|
err = translateSigned(m, z)
|
|
if err == nil {
|
|
t.Fatalf("no error from translating invalid table")
|
|
}
|
|
}
|