From 1a068cf0e2ce04e744c5d2a6985776f740762916 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 29 Sep 2021 11:16:10 -0500 Subject: [PATCH 1/7] added type support for looker; added intercept for yellowfin typelen query --- pg/protocol.go | 18 ++++++++++++++++++ pg/type.go | 39 +++++++++++++++++++++++++++++++++++---- server/pg.go | 15 +++++++++++++-- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/pg/protocol.go b/pg/protocol.go index e58540e1d..3ebc04934 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -32,6 +32,7 @@ import ( "github.com/molecula/featurebase/v2/pg/message" "github.com/molecula/featurebase/v2/sql" + "github.com/molecula/featurebase/v2/vprint" "github.com/pkg/errors" "vitess.io/vitess/go/vt/sqlparser" ) @@ -269,6 +270,7 @@ const ( pgSelect1 PgType = 'h' pgSchema PgType = 'i' pgBegin PgType = 'j' + pgTypeLen PgType = 'k' ) type Portal struct { @@ -298,6 +300,8 @@ func (p *Portal) Bind() { var lookPQL = regexp.MustCompile(`\[.*\].*\)\z`) +const POSTGRESLENSQL = `SELECT t.typlen FROM pg_catalog.pg_type t, pg_catalog.pg_namespace n WHERE t.typnamespace=n.oid AND t.typname='name' AND n.nspname='pg_catalog'` + func (p *Portal) Parse(data []byte) { p.queryStart = time.Now() queryStr := string(bytes.Trim(data, "\x00")) @@ -336,6 +340,10 @@ func (p *Portal) Parse(data []byte) { if strings.Contains(strings.ToLower(query.SQL), "select 1") { p.pgspecial = pgSelect1 p.Name = "SELECT" + } else if strings.Contains(queryStr, POSTGRESLENSQL) { + vprint.VV("YELLOWFIN Len") + p.pgspecial = pgTypeLen + p.Name = "SELECT" } else { switch query.SQLType { case sql.SQLTypeSet: @@ -565,6 +573,16 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) { case pgBegin: p.Name = "BEGIN" + case pgTypeLen: + rowDescription, e := p.Encoder.EncodeColumn("typelen", int32(21), 2) + if e != nil { + err = e + return + } + p.Add(rowDescription) + mesg := "64" + dataRow, _ := p.Encoder.TextRow(mesg) + p.Add(dataRow) } //maybe add in the number of items in select clause diff --git a/pg/type.go b/pg/type.go index 67e56d3e7..9952f5acd 100644 --- a/pg/type.go +++ b/pg/type.go @@ -20,18 +20,26 @@ import "github.com/molecula/featurebase/v2/pg/message" type Type struct { // I am not entirely sure what should be in here long term. // For now, I am just going to leave it like this. - id int32 + Id int32 + Typelen int16 } // TypeCharoid is a postgres type for text. -var TypeCharoid = Type{id: 18} +// found in postgres source src/include/catalog/pg_type.h +var TypeCharoid = Type{Id: 18, Typelen: -1} +var TypeNAMEOID = Type{Id: 19, Typelen: 64} +var TypeINT4OID = Type{Id: 23, Typelen: 4} +var TypeTEXTOID = Type{Id: 25, Typelen: -1} +var TypeFLOAT8OID = Type{Id: 701, Typelen: 8} // TypeData is a type containing raw postgres wire type information. +/* type TypeData struct { TypeID int32 TypeLen int16 TypeModifier int32 } +*/ // TypeEngine is a system for managing types. // This is necessary for compound types like arrays which need ID generation. @@ -45,9 +53,32 @@ type PrimitiveTypeEngine struct{} // TranslateType translates a type to a column description. func (pte PrimitiveTypeEngine) TranslateType(t Type) (message.ColumnDescription, error) { + var TypeLen int16 + var TypeID int32 + switch t { + case TypeCharoid: + TypeID = TypeCharoid.Id + TypeLen = TypeCharoid.Typelen + case TypeNAMEOID: + TypeID = TypeNAMEOID.Id + TypeLen = TypeNAMEOID.Typelen + case TypeINT4OID: + TypeID = TypeINT4OID.Id + TypeLen = TypeINT4OID.Typelen + case TypeTEXTOID: + TypeID = TypeTEXTOID.Id + TypeLen = TypeTEXTOID.Typelen + case TypeFLOAT8OID: + TypeID = TypeFLOAT8OID.Id + TypeLen = TypeFLOAT8OID.Typelen + //case + default: // treat like TypeCharoid: + TypeID = TypeCharoid.Id + TypeLen = TypeCharoid.Typelen + } return message.ColumnDescription{ - TypeID: t.id, // just charoid for now; as far as I can tell most implementations do not really use this - TypeLen: -1, // vdsm had 4. . . but the spec says this should be negative + TypeID: TypeID, + TypeLen: TypeLen, TypeModifier: -1, Mode: 0, // send as text }, nil diff --git a/server/pg.go b/server/pg.go index 533ca0b43..230541b87 100644 --- a/server/pg.go +++ b/server/pg.go @@ -28,6 +28,7 @@ import ( pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pg" + "github.com/molecula/featurebase/v2/sql2" //"github.com/molecula/featurebase/v2/pg" "github.com/molecula/featurebase/v2/pql" @@ -336,6 +337,15 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error return nil } +// TODO(twg) move this to a better area +func getPgType(sql2type string) pg.Type { + ret := pg.TypeCharoid + switch sql2type { + case sql2.DataTypeInt: + ret = pg.TypeINT4OID + } + return ret +} func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error { //TODO(twg) writeHeader //TODO(twg) writeColumns @@ -348,9 +358,10 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error { //TODO (twg) types:=rows.Types() headers := make([]pg.ColumnInfo, len(columns)) for i, column := range columns { + pgType := getPgType(column.Type) headers[i] = pg.ColumnInfo{ Name: column.Name, - Type: pg.TypeCharoid, //TODO(twg) types[i] + Type: pgType, } } err := w.WriteHeader(headers...) @@ -481,7 +492,7 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { return pgWriteGroupCount(w, result) case pb.ToRowser: // we should avoid protobuf where we can... return pgWriteRowser(w, result) - case *pilosa.StmtRows: // we should avoid protobuf where we can... + case *pilosa.StmtRows: return pgWriteStmtRows(w, result) case uint64: err := w.WriteHeader(pg.ColumnInfo{ From caee5680c317ac05193fc2c5dcf06dea92740cff Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 29 Sep 2021 14:22:23 -0500 Subject: [PATCH 2/7] wip --- server/pg.go | 11 ++++++++++- storage/config.go | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/pg.go b/server/pg.go index 230541b87..70ecdb074 100644 --- a/server/pg.go +++ b/server/pg.go @@ -418,6 +418,14 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error { } return nil } +func getPgTypeFromColumnInfo(sql2type string) pg.Type { + ret := pg.TypeCharoid + switch sql2type { + case sql2.DataTypeInt: + ret = pg.TypeINT4OID + } + return ret +} func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error { var data []string @@ -427,7 +435,8 @@ func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error { for i, h := range row.Headers { headers[i] = pg.ColumnInfo{ Name: h.Name, - Type: pg.TypeCharoid, + Type: pg.TypeCharoid, // TODO(twg) this needs to be updated with type + // information so it works from psql client } } err := w.WriteHeader(headers...) diff --git a/storage/config.go b/storage/config.go index e578ab9b9..05882db04 100644 --- a/storage/config.go +++ b/storage/config.go @@ -37,6 +37,6 @@ type Config struct { func NewDefaultConfig() *Config { return &Config{ Backend: DefaultBackend, - FsyncEnabled: true, + FsyncEnabled: false, } } From 34dad863e26d2291f72aae366566f6b176724d9d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 30 Sep 2021 15:49:59 -0500 Subject: [PATCH 3/7] wip --- go.mod | 2 +- pg/protocol.go | 3 ++- rbf/cfg/cfg.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ff7c1d715..42c4480a0 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 - gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v2 v2.3.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/pg/protocol.go b/pg/protocol.go index 3ebc04934..bd58cd2d0 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -331,7 +331,7 @@ func (p *Portal) Parse(data []byte) { } if len(queryStr) > 2 { - + vprint.VV("SQL=[%v]", queryStr) query, err := p.mapper.MapSQL(queryStr) if err != nil { return @@ -574,6 +574,7 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) { case pgBegin: p.Name = "BEGIN" case pgTypeLen: + vprint.VV("adding typelen message") rowDescription, e := p.Encoder.EncodeColumn("typelen", int32(21), 2) if e != nil { err = e diff --git a/rbf/cfg/cfg.go b/rbf/cfg/cfg.go index 5f8cd9345..fb0c034f2 100644 --- a/rbf/cfg/cfg.go +++ b/rbf/cfg/cfg.go @@ -56,7 +56,7 @@ func NewDefaultConfig() *Config { MaxWALSize: DefaultMaxWALSize, MinWALCheckpointSize: DefaultMinWALCheckpointSize, MaxWALCheckpointSize: DefaultMaxWALCheckpointSize, - FsyncEnabled: true, + FsyncEnabled: false, // CI passed with 20. 50 was too big for CI, even on X-large instances. // For now we default to 0, which means use sync.Pool. From c4c7d91bf016b21825a81faece1fb859665a605d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 12 Oct 2021 10:24:19 -0500 Subject: [PATCH 4/7] make linker happy --- server/pg.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/pg.go b/server/pg.go index 70ecdb074..672a07d77 100644 --- a/server/pg.go +++ b/server/pg.go @@ -427,6 +427,8 @@ func getPgTypeFromColumnInfo(sql2type string) pg.Type { return ret } +var _ = getPgTypeFromColumnInfo //make linter happy for this function will be needed in future + func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error { var data []string return result.ToRows(func(row *pb.RowResponse) error { From e4e0d13837107fbf36b5a85f6f69385d16cc36a9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 12 Oct 2021 10:40:32 -0500 Subject: [PATCH 5/7] go mod tidy correction? --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 42c4480a0..ff7c1d715 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/grpc v1.28.0 - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.3.0 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 sigs.k8s.io/yaml v1.2.0 // indirect From 5d57d361f42a1f570f377e3f0083b211a305ef36 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 12 Oct 2021 10:50:03 -0500 Subject: [PATCH 6/7] quite down --- pg/protocol.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pg/protocol.go b/pg/protocol.go index bd58cd2d0..ec1084a23 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -32,7 +32,6 @@ import ( "github.com/molecula/featurebase/v2/pg/message" "github.com/molecula/featurebase/v2/sql" - "github.com/molecula/featurebase/v2/vprint" "github.com/pkg/errors" "vitess.io/vitess/go/vt/sqlparser" ) @@ -331,7 +330,6 @@ func (p *Portal) Parse(data []byte) { } if len(queryStr) > 2 { - vprint.VV("SQL=[%v]", queryStr) query, err := p.mapper.MapSQL(queryStr) if err != nil { return @@ -341,7 +339,6 @@ func (p *Portal) Parse(data []byte) { p.pgspecial = pgSelect1 p.Name = "SELECT" } else if strings.Contains(queryStr, POSTGRESLENSQL) { - vprint.VV("YELLOWFIN Len") p.pgspecial = pgTypeLen p.Name = "SELECT" } else { @@ -574,7 +571,6 @@ func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) { case pgBegin: p.Name = "BEGIN" case pgTypeLen: - vprint.VV("adding typelen message") rowDescription, e := p.Encoder.EncodeColumn("typelen", int32(21), 2) if e != nil { err = e From 4d4f64a33989a1e782a19e0515b57686ddd5ca4d Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 12 Oct 2021 11:21:14 -0500 Subject: [PATCH 7/7] address ben's comments --- pg/type.go | 10 ---------- rbf/cfg/cfg.go | 2 +- server/pg.go | 7 ++++--- storage/config.go | 2 +- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pg/type.go b/pg/type.go index 9952f5acd..98ea4348f 100644 --- a/pg/type.go +++ b/pg/type.go @@ -32,15 +32,6 @@ var TypeINT4OID = Type{Id: 23, Typelen: 4} var TypeTEXTOID = Type{Id: 25, Typelen: -1} var TypeFLOAT8OID = Type{Id: 701, Typelen: 8} -// TypeData is a type containing raw postgres wire type information. -/* -type TypeData struct { - TypeID int32 - TypeLen int16 - TypeModifier int32 -} -*/ - // TypeEngine is a system for managing types. // This is necessary for compound types like arrays which need ID generation. type TypeEngine interface { @@ -71,7 +62,6 @@ func (pte PrimitiveTypeEngine) TranslateType(t Type) (message.ColumnDescription, case TypeFLOAT8OID: TypeID = TypeFLOAT8OID.Id TypeLen = TypeFLOAT8OID.Typelen - //case default: // treat like TypeCharoid: TypeID = TypeCharoid.Id TypeLen = TypeCharoid.Typelen diff --git a/rbf/cfg/cfg.go b/rbf/cfg/cfg.go index fb0c034f2..5f8cd9345 100644 --- a/rbf/cfg/cfg.go +++ b/rbf/cfg/cfg.go @@ -56,7 +56,7 @@ func NewDefaultConfig() *Config { MaxWALSize: DefaultMaxWALSize, MinWALCheckpointSize: DefaultMinWALCheckpointSize, MaxWALCheckpointSize: DefaultMaxWALCheckpointSize, - FsyncEnabled: false, + FsyncEnabled: true, // CI passed with 20. 50 was too big for CI, even on X-large instances. // For now we default to 0, which means use sync.Pool. diff --git a/server/pg.go b/server/pg.go index 672a07d77..c2522f286 100644 --- a/server/pg.go +++ b/server/pg.go @@ -419,12 +419,13 @@ func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error { return nil } func getPgTypeFromColumnInfo(sql2type string) pg.Type { - ret := pg.TypeCharoid switch sql2type { case sql2.DataTypeInt: - ret = pg.TypeINT4OID + return pg.TypeINT4OID + default: + return pg.TypeCharoid + } - return ret } var _ = getPgTypeFromColumnInfo //make linter happy for this function will be needed in future diff --git a/storage/config.go b/storage/config.go index 05882db04..e578ab9b9 100644 --- a/storage/config.go +++ b/storage/config.go @@ -37,6 +37,6 @@ type Config struct { func NewDefaultConfig() *Config { return &Config{ Backend: DefaultBackend, - FsyncEnabled: false, + FsyncEnabled: true, } }