mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 09:05:55 +00:00
Merge pull request #1954 from kuba--/reopen
TranslateFile - reopen the same instance
This commit is contained in:
Matthew Jaffee
GitHub
commit
d7d52d6b4e
4 changed files with 168 additions and 11 deletions
1
go.mod
1
go.mod
|
|
@ -3,6 +3,7 @@ module github.com/pilosa/pilosa
|
|||
replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v0.3.1 // indirect
|
||||
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d
|
||||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
|
||||
|
|
|
|||
21
go.sum
21
go.sum
|
|
@ -1,7 +1,10 @@
|
|||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI=
|
||||
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
|
||||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
|
||||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
|
|
@ -33,8 +36,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
|||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI=
|
||||
github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
|
||||
github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA=
|
||||
github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
|
||||
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
|
||||
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
|
|
@ -47,6 +48,7 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
|
|||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
|
|
@ -64,19 +66,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
|
|||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo=
|
||||
github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
|
|
@ -101,17 +99,20 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
|||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
|
||||
github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk=
|
||||
github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
|
||||
github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw=
|
||||
github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ func (s *TranslateFile) Open() (err error) {
|
|||
return errors.Wrapf(err, "open file %s", s.Path)
|
||||
}
|
||||
s.w = bufio.NewWriter(s.file)
|
||||
s.n = 0
|
||||
|
||||
// Memory map data file.
|
||||
if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil {
|
||||
|
|
|
|||
|
|
@ -758,6 +758,156 @@ func TestTranslateFile_ReassignPrimaryTranslateStore(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestTranslateFile_ReopenTheSameInstance(t *testing.T) {
|
||||
s := MustOpenTranslateFile()
|
||||
defer s.MustClose()
|
||||
|
||||
// First translation should start id at zero.
|
||||
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Next translation on the same index should move to one.
|
||||
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{2}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Translation on a different index restarts at 0.
|
||||
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Ensure that string values can be looked up by ID.
|
||||
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "bar" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
if value, err := s.TranslateColumnToString("IDX0", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "foo" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
// Ensure that non-existent values return "".
|
||||
if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
if err := s.TranslateFile.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s.MustOpen()
|
||||
|
||||
// Ensure translation is still correct after reopen.
|
||||
if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Ensure translation is still correct after reopen.
|
||||
if value, err := s.TranslateColumnToString("IDX0", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "bar" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
// Next translation on the same index should move to one.
|
||||
if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{3}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
if err := s.TranslateFile.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s.MustOpen()
|
||||
|
||||
// First translation should start id at zero.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Next translation on the same index should move to one.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{2}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Translation on a different index restarts at 0.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Translation on a different field restarts at 0.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Ensure that string values can be looked up by ID.
|
||||
if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "bar" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
// Ensure that non-existent values return blank.
|
||||
if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
if err := s.TranslateFile.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// Reopen the store.
|
||||
s.MustOpen()
|
||||
|
||||
// Translation on a different field restarts at 0.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{1}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
// Ensure that string values can be looked up by ID.
|
||||
if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if value != "bar" {
|
||||
t.Fatalf("unexpected value: %s", value)
|
||||
}
|
||||
|
||||
// Translate new row and increment sequence.
|
||||
if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(ids, []uint64{3}) {
|
||||
t.Fatalf("unexpected id: %#v", ids)
|
||||
}
|
||||
|
||||
if err := s.TranslateFile.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) {
|
||||
const batchSize = 1000
|
||||
|
||||
|
|
@ -836,10 +986,14 @@ func (t *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser
|
|||
|
||||
func MustOpenTranslateFile() *TranslateFile {
|
||||
s := NewTranslateFile()
|
||||
s.MustOpen()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *TranslateFile) MustOpen() {
|
||||
if err := s.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *TranslateFile) Close() error {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue