mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
* unifying idk and featurebase: first pass * resolved conflict with master for gitignore & dockerignore * deleted binaries that were accidentally pushed to git * combined gitlab jobs for idk & featurebase * run go fmt for idk * updated ssh env variable, and made docker password variable in gitlab env variables * fixed typo assigning variable name * trying to fix docker login error * trying a different solution for docker password * pass registry * fixed docker login * updated paths for idk * exclude idk tests from featurebase test run * fix vendor error * update certificates * grpc needs to be in version 1.38 genproto, which is imported by big query updates the grpc version to 1.47.0 grpc 1.47.0 causes etcd to deadlock when calling etcd.Close() the fix is to have a replace in go.mod to specify a specific grpc version * run go mod tidy * go mod * run go mod tidy * exclude bigquery since it is causing issues and undo grpc replace in go.mod * fix grpc version * fix formatting error * update formatting * attempt to fix formatting * update path for code coverage * update to use current branch binaries, not master * fix for building idk - path updates * udpate path for binaries * update job dependecies * update docker idk tests to use the current branch registry * update stages for jobs * updated job dependencies * not allow idk s3 dump to fail since it is a dependency for integration tests * update dependecy for idk tests * update paths for idk build and code coverage * download featurebase binary from s3 * pass branch name to all setup scripts * change to current branch instead of master * updated sonarcloud * sonarcloud fix and branch name fix * trying to speed up pipeline run time * update stage * branch name fix + sonar cloud * sonarcloud
383 lines
12 KiB
Text
383 lines
12 KiB
Text
# FEATUREBASE HOST CONFIGURATION
|
|
#
|
|
# Uncomment when/where appropriate
|
|
|
|
# ==============================================================================
|
|
# Use advertise to specify the address advertised by the server to other nodes
|
|
# in the cluster and to clients via /status endpoint. Host defaults to IP
|
|
# address represented by bind parameter with network port.
|
|
#
|
|
# advertise = <IP Address>:10101
|
|
# advertise-grpc = <IP Address>:20101
|
|
|
|
|
|
|
|
# "long-query-time" represents duration of time that will trigger log and stat
|
|
# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds
|
|
#
|
|
# long-query-time = "10s"
|
|
|
|
|
|
|
|
# Unique name for node in cluster. This is just a human-readable label for
|
|
# convenience and not used by any underlying logic.
|
|
|
|
# name = "pilosa1"
|
|
|
|
|
|
|
|
# # Host:Port where Featurebase server listens for HTTP requests.
|
|
# # Default is localhost:10101
|
|
# #
|
|
# bind = "pilosa1:10101"
|
|
|
|
|
|
|
|
# # The address and port featurebase will listen to for all GRPC connections
|
|
# # Ex. python-molecula, grafana for queries, etc.
|
|
# #
|
|
# bind-grpc = "localhost:20101"
|
|
|
|
|
|
|
|
# Directory to store Featurebase data files
|
|
# data-dir = "/var/lib/molecula"
|
|
|
|
|
|
# ==============================================================================
|
|
# CORS (Cross-Origin Resource Sharing) Allowed Origins
|
|
# List of allowed origin URIs for CORS
|
|
#
|
|
# [handler]
|
|
# allowed-origins = ["https://myapp.com", "https://myapp.org"]
|
|
|
|
|
|
|
|
# Path to the log file
|
|
# log-path = "/var/log/molecula/featurebase.log"
|
|
|
|
|
|
|
|
# Verbose - Enable verbose logging. Valid options are true or false.
|
|
# Set to true only when debugging as directed by Molecula engineers.
|
|
#
|
|
# verbose = true
|
|
|
|
|
|
|
|
# Soft limit on max number of files featurebase will keep open simultaneously.
|
|
# When past this limit, featurebase will only keep files open for as long as is
|
|
# needed to write updates.
|
|
#
|
|
# max-file-count = 900000
|
|
|
|
|
|
|
|
# Maximum number of active memory maps featurebase will use for fragment files.
|
|
# Actual total usage may be slightly higher.
|
|
# Best practice is to set this to ~10% lower than your system's max map count.
|
|
# See sysctl vm.max_map_count in Linux.
|
|
#
|
|
# max-map-count = 900000
|
|
|
|
|
|
|
|
# Max Writes Per Request - Max number of mutating commands allowed per request.
|
|
# This includes Set, Clear, ClearRow, and Store
|
|
#
|
|
# max-writes-per-request = 5000
|
|
|
|
|
|
|
|
# The following option sets the maximum number of queries that are maintained
|
|
# for the /query-history endpoint.
|
|
# This parameter is per-node, and the result combines the history from all nodes.
|
|
#
|
|
# query-history-length = 100
|
|
|
|
|
|
|
|
# External database to connect to for `ExternalLookup` queries.
|
|
# lookup-db-dsn = "postgres://localhost:5432/db"
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# For cluster stanza, "name" represents name for cluster. Must be same on all
|
|
# nodes in cluster. "replicas" represents number of hosts each piece of data
|
|
# should be stored on. Must be greater than or equal to 1 & less than or equal
|
|
# to number of nodes in cluster.
|
|
# [cluster]
|
|
# name = "cluster1"
|
|
# replicas = 1
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# [etcd]
|
|
# etcd is the tool Featurebase uses for node-to-node, intra-cluster
|
|
# communication. etcd is embedded in the featurebase cluster rather than
|
|
# running as a separate instance.
|
|
# It's important to configure this correctly for your network and nodes, and
|
|
# that it is consistent across all nodes.
|
|
#
|
|
# The easiest setup can be used when all nodes can reach all other nodes via a
|
|
# local subnet:
|
|
# listen-peer-address = advertise-peer-address
|
|
# = (what's in the initial-cluster-list)
|
|
# = the nodes ip address (which can be reached by every
|
|
# other node
|
|
# (localhost:10401 would not work for this, as each node can't reach that)
|
|
#
|
|
# If each node is separated by a proxy, or must be reached via url / dns, you
|
|
# will need to use a more complicated setup:
|
|
# listen-peer-address = the nodes local ip address
|
|
# (specific ip, localhost, or 0.0.0.0 for all
|
|
# local ip's)
|
|
# advertise-peer-address = the nodes ip address, reachable by all other nodes
|
|
# (This address should also be included in
|
|
# initital-cluster-list)
|
|
# in this case, you specify a different url/ip for listen-peer and
|
|
# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case)
|
|
# you use a url for advertise. In each of these cases, you should set listen
|
|
# to the local ip, and you set advertise = to how each other node connects to
|
|
# this node, and you also use this same address in the initial cluster.
|
|
# The key here is that initial-cluster has to include the same node name and
|
|
# advertise-peer address as the node it's on (edited)
|
|
|
|
|
|
|
|
# for additional assistance, and for help with config issues,
|
|
# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster
|
|
# that a new node should join when adding nodes to cluster.
|
|
#
|
|
# cluster-url = "http://localhost:10401"
|
|
|
|
|
|
|
|
# Address and port to bind to for client communication
|
|
# listen-client-address = "http://localhost:10401"
|
|
|
|
|
|
|
|
# Address and port to bind to for peer communication
|
|
# listen-peer-address = "http://localhost:10301"
|
|
|
|
|
|
|
|
# Comma-separated list of node=address pairs that makes up initial cluster when
|
|
# first started. In each pair, "node" value (left side of = ) should match
|
|
# name of node specified by "name" configuration parameter
|
|
#
|
|
# initial-cluster = "featurebase1=http://localhost:10301"
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# Profile Block Rate - Block Rate is passed directly to Go's
|
|
# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1
|
|
# per rate nanoseconds. A value of "1" samples every event, and 0 disables
|
|
# profiling.
|
|
#
|
|
# block-rate = 10000000
|
|
|
|
# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's
|
|
# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled.
|
|
#
|
|
# mutex-fraction = 100
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# PostgreSQL Section
|
|
# [postgres]
|
|
#
|
|
# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint.
|
|
# No PostgreSQL endpoint will be exposed unless a bind address is specified.
|
|
# Requires Molecula v3.0 or newer.
|
|
#
|
|
# bind = "localhost:55432"
|
|
|
|
|
|
|
|
# The PostgreSQL endpoint has support for a connection limit.
|
|
# This is generally not necessary, so it is disabled by default.
|
|
#
|
|
# connection-limit = 10000
|
|
|
|
|
|
|
|
# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint
|
|
# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should
|
|
# typically be sufficient, but may be exceeded if a client sends an unusually
|
|
# large amount of configuration data. Oversized startup packets are typically
|
|
# caused by connecting with a different protocol, e.g. HTTP.
|
|
#
|
|
# max-startup-size = 10000000
|
|
|
|
|
|
|
|
# PostgreSQL Timeouts
|
|
# In order to detect stalled clients, the PostgreSQL endpoint has connection
|
|
# read and write timeouts. There is also a startup timeout, which is used for
|
|
# connection setup. The read timeout does not impact idle connections. Idle
|
|
# connections will only be closed by the server if TCP keepalive reports a
|
|
# break in the connection. TCP keepalives use the default configuration
|
|
# provided by the host.
|
|
# Caution: Due to a limitation of the PostgreSQL wire protocol,
|
|
# raising the write timeout may delay the shutdown of a featurebase node.
|
|
#
|
|
# startup-timeout = "20s"
|
|
# read-timeout = "20s"
|
|
# write-timeout - "20s"
|
|
|
|
|
|
|
|
# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is
|
|
# structured the same as the TLS configuration for Featurebase's other endpoints,
|
|
# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint,
|
|
# Featurebase will reject unsecured connections.
|
|
# [postgres.tls]
|
|
# certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt"
|
|
# key = "/Users/souhailanoor/tls/out/auth.mybusiness.com.key"
|
|
# ca-certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt"
|
|
# enable-client-verification = true
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is
|
|
# calculated periodically in the background and accessed by the UI/usage
|
|
# endpoint. Since this disk scan can take a long and unpredictable amount of
|
|
# time, its timing behavior is specified in a relative, rather than absolute
|
|
# sense. That is, the duty cycle sets the percentage of time that is spent
|
|
# recalculating this cache. This setting affects the results received from
|
|
# the "/ui/usage" http endpoint, as well as all data file and memory usage
|
|
# values and graphs on the webui "tables" page
|
|
|
|
# Special considerations:
|
|
# * If disk usage can be calculated quickly (less than 5 seconds), fresh
|
|
# results will be calculated when accessed
|
|
# * When disk usage takes longer to calculate, there is a minimum of one
|
|
# hour wait between cache recalculations
|
|
# Setting this value to 0 will completely disable the calculation of disk usage
|
|
#
|
|
# usage-duty-cycle = 20
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# Use [metric] stanza to define attributes for monitoring.
|
|
# [metric]
|
|
# Specify which service to use for collecting metrics. Valid options are:
|
|
# "statsd", "expvar", "prometheus", "none"
|
|
#
|
|
# service = "prometheus"
|
|
|
|
|
|
|
|
# Remote host to send statsd metrics to.
|
|
# host = "localhost:8125"
|
|
|
|
|
|
|
|
# The interval to send statsd metrics.
|
|
# poll-interval = "10s"
|
|
|
|
|
|
|
|
# Debugging flag to enable to send diagnostic information to Featurebase
|
|
# developers.
|
|
#
|
|
# diagnostics = false
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# TLS Certificate Section - Path to TLC certificate used for service HTTPS.
|
|
# Suffix should contain .crt or .pem
|
|
|
|
[tls]
|
|
certificate = "/testdata/certs/localhost.crt"
|
|
key = "/testdata/certs/localhost.key"
|
|
|
|
# ==============================================================================
|
|
# Tracing Section
|
|
# [tracing]
|
|
#
|
|
# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting",
|
|
# or "remote". Set to 'off' to disable tracing completely.
|
|
#
|
|
# sampler-type = "remote"
|
|
|
|
|
|
# Jaeger sampler parameter (number)
|
|
# sampler-param = 0.001
|
|
|
|
|
|
|
|
# Tracing Agent Host:Port
|
|
# agent-host-port = "localhost:6831"
|
|
|
|
|
|
|
|
# ==============================================================================
|
|
# Configuration for the RBF storage format.
|
|
# [rbf]
|
|
# Maximum size for each RBF database file.
|
|
# Allocates virtual memory but does not preallocate physical disk space.
|
|
# If you get into the range where you have 16000 shards on a single node
|
|
# (across all indexes), you will need to lower this in order to not run out of
|
|
# virtual address space.
|
|
#
|
|
# max-db-size = 4294967296
|
|
|
|
|
|
|
|
# Maximum size for each RBF WAL file.
|
|
# Allocates virtual memory but does not preallocate physical disk space.
|
|
# This is the same as max-db-size, but for the write-ahead log. If you set it
|
|
# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely
|
|
# condense these options in the future).
|
|
#
|
|
# max-wal-size = 4294967296
|
|
|
|
|
|
|
|
# Minimum WAL size before WAL pages can be copied to the main database file.
|
|
# min-wal-checkpoint-size = 1048576
|
|
|
|
|
|
|
|
# Maximum WAL size before transactions are halted to copy WAL pages to the
|
|
# main database file.
|
|
#
|
|
# max-wal-checkpoint-size = 2147483648
|
|
|
|
|
|
# ==============================================================================
|
|
# [storage]
|
|
# Sync all changes to the file system.
|
|
# Should not be changed in production systems unless you know what you are
|
|
# doing - Should always be on unless testing or possibly while performing a
|
|
# bulk import and you are not worried about data loss
|
|
#
|
|
# fsync = true
|
|
|
|
|
|
# ==============================================================================
|
|
# Enable/Disable AuthN/AuthZ for featurebase
|
|
# Can choose identity provider, pass authorize and user-info endpoints, and client id
|
|
[auth]
|
|
enable = true
|
|
client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
|
|
client-secret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
|
|
authorize-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/authorize"
|
|
token-url="http://fakeidp:10101/token"
|
|
group-endpoint-url = "http://fakeidp:10101/groups"
|
|
logout-url = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
|
|
scopes = ["https://graph.microsoft.com/.default", "offline_access"]
|
|
secret-key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
|
|
permissions = "/testdata/permissions.yaml"
|
|
query-log-path = "query-log-test.log"
|
|
redirect-base-url = "https://localhost:10101"
|