diff --git a/go.mod b/go.mod index 053cee36a..397d0e267 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/pilosa/pilosa replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 require ( + github.com/BurntSushi/toml v0.3.1 // indirect github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect diff --git a/go.sum b/go.sum index 1e00ccec0..a1541510b 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,10 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -33,8 +36,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= -github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -47,6 +48,7 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -64,19 +66,15 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo= -github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec= -github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA= -github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 h1:YDeskXpkNDhPdWN3REluVa46HQOVuVkjkd2sWnrABNQ= github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -101,17 +99,20 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw= -github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/translate.go b/translate.go index 3e4245ae1..94eb57186 100644 --- a/translate.go +++ b/translate.go @@ -167,6 +167,7 @@ func (s *TranslateFile) Open() (err error) { return errors.Wrapf(err, "open file %s", s.Path) } s.w = bufio.NewWriter(s.file) + s.n = 0 // Memory map data file. if s.data, err = syscall.Mmap(int(s.file.Fd()), 0, s.mapSize, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { diff --git a/translate_test.go b/translate_test.go index a2f0b0268..4fa977e62 100644 --- a/translate_test.go +++ b/translate_test.go @@ -758,6 +758,156 @@ func TestTranslateFile_ReassignPrimaryTranslateStore(t *testing.T) { }) } +func TestTranslateFile_ReopenTheSameInstance(t *testing.T) { + s := MustOpenTranslateFile() + defer s.MustClose() + + // First translation should start id at zero. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + if value, err := s.TranslateColumnToString("IDX0", 1); err != nil { + t.Fatal(err) + } else if value != "foo" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return "". + if value, err := s.TranslateColumnToString("IDX0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // Ensure translation is still correct after reopen. + if ids, err := s.TranslateColumnsToUint64("IDX1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure translation is still correct after reopen. + if value, err := s.TranslateColumnToString("IDX0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateColumnsToUint64("IDX0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + s.MustOpen() + + // First translation should start id at zero. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"foo"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Next translation on the same index should move to one. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{2}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different index restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX1", "FIELD0", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Ensure that non-existent values return blank. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 1000); err != nil { + t.Fatal(err) + } else if value != "" { + t.Fatalf("unexpected value: %s", value) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } + // Reopen the store. + s.MustOpen() + + // Translation on a different field restarts at 0. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD1", []string{"bar"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{1}) { + t.Fatalf("unexpected id: %#v", ids) + } + + // Ensure that string values can be looked up by ID. + if value, err := s.TranslateRowToString("IDX0", "FIELD0", 2); err != nil { + t.Fatal(err) + } else if value != "bar" { + t.Fatalf("unexpected value: %s", value) + } + + // Translate new row and increment sequence. + if ids, err := s.TranslateRowsToUint64("IDX0", "FIELD0", []string{"baz"}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ids, []uint64{3}) { + t.Fatalf("unexpected id: %#v", ids) + } + + if err := s.TranslateFile.Close(); err != nil { + panic(err) + } +} + func BenchmarkTranslateFile_TranslateColumnsToUint64(b *testing.B) { const batchSize = 1000 @@ -836,10 +986,14 @@ func (t *TranslateFile) Reader(ctx context.Context, offset int64) (io.ReadCloser func MustOpenTranslateFile() *TranslateFile { s := NewTranslateFile() + s.MustOpen() + return s +} + +func (s *TranslateFile) MustOpen() { if err := s.Open(); err != nil { panic(err) } - return s } func (s *TranslateFile) Close() error {