diff --git a/Makefile b/Makefile index e2f659cf2..55d1332ba 100644 --- a/Makefile +++ b/Makefile @@ -104,10 +104,19 @@ build: # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase" - cp NOTICE README.md LICENSE build/featurebase$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID) tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ @echo Created release build: build/featurebase-$(VERSION_ID).tar.gz +test-release-build: docker-build + mv build/featurebase-$(VERSION_ID).tar.gz install/ + cd install && docker build -t featurebase:test_installation \ + -f test_installation.Dockerfile \ + --build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz . + mv install/featurebase-$(VERSION_ID).tar.gz build/ + docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + featurebase:test_installation + # Error out if there are untracked changes in Git check-clean: ifndef SKIP_CHECK_CLEAN @@ -220,7 +229,7 @@ docker-build: vendor docker create --name featurebase-build featurebase:build mkdir -p build/featurebase-$(VERSION_ID) docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) - cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID) docker rm featurebase-build tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ diff --git a/NOTICE b/NOTICE index 594272a27..cb62dfdca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,23 +1,12 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= -The file /pilosa/lru/lru.go contains a redistribution of lru +The file /lru/lru.go contains a redistribution of lru (github.com/golang/groupcache/lru); the license follows: Copyright 2013 Google Inc. diff --git a/install/featurebase.conf b/install/featurebase.conf new file mode 100644 index 000000000..73895ed6d --- /dev/null +++ b/install/featurebase.conf @@ -0,0 +1,373 @@ +# FEATUREBASE HOST CONFIGURATION +# +# Uncomment when/where appropriate + +# ============================================================================== +# Use advertise to specify the address advertised by the server to other nodes +# in the cluster and to clients via /status endpoint. Host defaults to IP +# address represented by bind parameter with network port. +# +# advertise = :10101 +# advertise-grpc = :20101 + + + +# "long-query-time" represents duration of time that will trigger log and stat +# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds +# +# long-query-time = "10s" + + + +# Unique name for node in cluster. This is just a human-readable label for +# convenience and not used by any underlying logic. +# +# name = "featurebase1" + + + +# Host:Port where Featurebase server listens for HTTP requests. +# Default is localhost:10101 +# +# bind = "localhost:10101" + + + +# The address and port featurebase will listen to for all GRPC connections +# Ex. python-molecula, grafana for queries, etc. +# +# bind-grpc = "0.0.0.0:20101" + + + +# Directory to store Featurebase data files +data-dir = "/var/lib/molecula" + + +# ============================================================================== +# CORS (Cross-Origin Resource Sharing) Allowed Origins +# List of allowed origin URIs for CORS +# +# [handler] +# allowed-origins = ["https://myapp.com", "https://myapp.org"] + + + +# Path to the log file +log-path = "/var/log/molecula/featurebase.log" + + + +# Verbose - Enable verbose logging. Valid options are true or false. +# Set to true only when debugging as directed by Molecula engineers. +# +# verbose = true + + + +# Soft limit on max number of files featurebase will keep open simultaneously. +# When past this limit, featurebase will only keep files open for as long as is +# needed to write updates. +# +# max-file-count = 900000 + + + +# Maximum number of active memory maps featurebase will use for fragment files. +# Actual total usage may be slightly higher. +# Best practice is to set this to ~10% lower than your system's max map count. +# See sysctl vm.max_map_count in Linux. +# +# max-map-count = 900000 + + + +# Max Writes Per Request - Max number of mutating commands allowed per request. +# This includes Set, Clear, ClearRow, and Store +# +# max-writes-per-request = 5000 + + + +# The following option sets the maximum number of queries that are maintained +# for the /query-history endpoint. +# This parameter is per-node, and the result combines the history from all nodes. +# +# query-history-length = 100 + + + +# External database to connect to for `ExternalLookup` queries. +# lookup-db-dsn = "postgres://localhost:5432/db" + + + +# ============================================================================== +# For cluster stanza, "name" represents name for cluster. Must be same on all +# nodes in cluster. "replicas" represents number of hosts each piece of data +# should be stored on. Must be greater than or equal to 1 & less than or equal +# to number of nodes in cluster. +# [cluster] +# name = "cluster1" +# replicas = 1 + + + +# ============================================================================== +# [etcd] +# etcd is the tool Featurebase uses for node-to-node, intra-cluster +# communication. etcd is embedded in the featurebase cluster rather than +# running as a separate instance. +# It's important to configure this correctly for your network and nodes, and +# that it is consistent across all nodes. +# +# The easiest setup can be used when all nodes can reach all other nodes via a +# local subnet: +# listen-peer-address = advertise-peer-address +# = (what's in the initial-cluster-list) +# = the nodes ip address (which can be reached by every +# other node +# (localhost:10401 would not work for this, as each node can't reach that) +# +# If each node is separated by a proxy, or must be reached via url / dns, you +# will need to use a more complicated setup: +# listen-peer-address = the nodes local ip address +# (specific ip, localhost, or 0.0.0.0 for all +# local ip's) +# advertise-peer-address = the nodes ip address, reachable by all other nodes +# (This address should also be included in +# initital-cluster-list) +# in this case, you specify a different url/ip for listen-peer and +# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) +# you use a url for advertise. In each of these cases, you should set listen +# to the local ip, and you set advertise = to how each other node connects to +# this node, and you also use this same address in the initial cluster. +# The key here is that initial-cluster has to include the same node name and +# advertise-peer address as the node it's on (edited) + + + +# for additional assistance, and for help with config issues, +# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster +# that a new node should join when adding nodes to cluster. +# +# cluster-url = "http://localhost:10401" + + + +# Address and port to bind to for client communication +# listen-client-address = "http://localhost:10401" + + + +# Address and port to bind to for peer communication +# listen-peer-address = "http://localhost:10301" + + + +# Comma-separated list of node=address pairs that makes up initial cluster when +# first started. In each pair, "node" value (left side of = ) should match +# name of node specified by "name" configuration parameter +# +# initial-cluster = "featurebase1=http://localhost:10301" + + + +# ============================================================================== +# Profile Block Rate - Block Rate is passed directly to Go's +# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 +# per rate nanoseconds. A value of "1" samples every event, and 0 disables +# profiling. +# +# block-rate = 10000000 + +# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's +# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled. +# +# mutex-fraction = 100 + + + +# ============================================================================== +# PostgreSQL Section +# [postgres] +# +# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint. +# No PostgreSQL endpoint will be exposed unless a bind address is specified. +# Requires Molecula v3.0 or newer. +# +# bind = "localhost:55432" + + + +# The PostgreSQL endpoint has support for a connection limit. +# This is generally not necessary, so it is disabled by default. +# +# connection-limit = 10000 + + + +# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint +# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should +# typically be sufficient, but may be exceeded if a client sends an unusually +# large amount of configuration data. Oversized startup packets are typically +# caused by connecting with a different protocol, e.g. HTTP. +# +# max-startup-size = 10000000 + + + +# PostgreSQL Timeouts +# In order to detect stalled clients, the PostgreSQL endpoint has connection +# read and write timeouts. There is also a startup timeout, which is used for +# connection setup. The read timeout does not impact idle connections. Idle +# connections will only be closed by the server if TCP keepalive reports a +# break in the connection. TCP keepalives use the default configuration +# provided by the host. +# Caution: Due to a limitation of the PostgreSQL wire protocol, +# raising the write timeout may delay the shutdown of a featurebase node. +# +# startup-timeout = "20s" +# read-timeout = "20s" +# write-timeout - "20s" + + + +# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is +# structured the same as the TLS configuration for Featurebase's other endpoints, +# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint, +# Featurebase will reject unsecured connections. +# [postgres.tls] +# certificate = "/srv/pilosa/certs/server.crt" +# key = "/srv/pilosa/certs/server.key" +# ca-certificate = "/srv/pilosa/certs/ca.crt" +# enable-client-verification = true + + + +# ============================================================================== +# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is +# calculated periodically in the background and accessed by the UI/usage +# endpoint. Since this disk scan can take a long and unpredictable amount of +# time, its timing behavior is specified in a relative, rather than absolute +# sense. That is, the duty cycle sets the percentage of time that is spent +# recalculating this cache. This setting affects the results received from +# the "/ui/usage" http endpoint, as well as all data file and memory usage +# values and graphs on the webui "tables" page + +# Special considerations: +# * If disk usage can be calculated quickly (less than 5 seconds), fresh +# results will be calculated when accessed +# * When disk usage takes longer to calculate, there is a minimum of one +# hour wait between cache recalculations +# Setting this value to 0 will completely disable the calculation of disk usage +# +# usage-duty-cycle = 20 + + + +# ============================================================================== +# Use [metric] stanza to define attributes for monitoring. +# [metric] +# Specify which service to use for collecting metrics. Valid options are: +# "statsd", "expvar", "prometheus", "none" +# +# service = "prometheus" + + + +# Remote host to send statsd metrics to. +# host = "localhost:8125" + + + +# The interval to send statsd metrics. +# poll-interval = "10s" + + + +# Debugging flag to enable to send diagnostic information to Featurebase +# developers. +# +# diagnostics = false + + + +# ============================================================================== +# TLS Certificate Section - Path to TLC certificate used for service HTTPS. +# Suffix should contain .crt or .pem +# +# [tls] +# certificate = "/srv/pilosa/certs/server.crt" +# TLS Certificate Key - Path to TLS certificate key for HTTPS. Suffix should +# be .key +# +# key = "/srv/pilosa/certs/server.key" + + + +# ============================================================================== +# Tracing Section +# [tracing] +# +# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", +# or "remote". Set to 'off' to disable tracing completely. +# +# sampler-type = "remote" + + +# Jaeger sampler parameter (number) +# sampler-param = 0.001 + + + +# Tracing Agent Host:Port +# agent-host-port = "localhost:6831" + + + +# ============================================================================== +# Configuration for the RBF storage format. +# [rbf] +# Maximum size for each RBF database file. +# Allocates virtual memory but does not preallocate physical disk space. +# If you get into the range where you have 16000 shards on a single node +# (across all indexes), you will need to lower this in order to not run out of +# virtual address space. +# +# max-db-size = 4294967296 + + + +# Maximum size for each RBF WAL file. +# Allocates virtual memory but does not preallocate physical disk space. +# This is the same as max-db-size, but for the write-ahead log. If you set it +# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely +# condense these options in the future). +# +# max-wal-size = 4294967296 + + + +# Minimum WAL size before WAL pages can be copied to the main database file. +# min-wal-checkpoint-size = 1048576 + + + +# Maximum WAL size before transactions are halted to copy WAL pages to the +# main database file. +# +# max-wal-checkpoint-size = 2147483648 + + +# ============================================================================== +# [storage] +# Sync all changes to the file system. +# Should not be changed in production systems unless you know what you are +# doing - Should always be on unless testing or possibly while performing a +# bulk import and you are not worried about data loss +# +# fsync = true + + +# ============================================================================== diff --git a/install/featurebase.debian.service b/install/featurebase.debian.service new file mode 100644 index 000000000..162264106 --- /dev/null +++ b/install/featurebase.debian.service @@ -0,0 +1,13 @@ +[Unit] +Description="Service for FeatureBase" +After=network.target + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/featurebase.redhat.service b/install/featurebase.redhat.service new file mode 100644 index 000000000..cf507d8b0 --- /dev/null +++ b/install/featurebase.redhat.service @@ -0,0 +1,12 @@ +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/test_installation.Dockerfile b/install/test_installation.Dockerfile new file mode 100644 index 000000000..5d9a93253 --- /dev/null +++ b/install/test_installation.Dockerfile @@ -0,0 +1,18 @@ +FROM fedora:rawhide + +RUN yum -y install systemd procps + +ARG release_tarball + +COPY $release_tarball . + +COPY test_installation.sh test_installation.sh + +RUN mkdir install && \ + tar -xf $release_tarball -C install --strip-components 1 && \ + cd install && \ + cp featurebase /usr/local/bin/featurebase && \ + cp featurebase.redhat.service /etc/systemd/system/featurebase.service && \ + cp featurebase.conf /etc/featurebase.conf + +CMD ["/bin/bash", "test_installation.sh"] diff --git a/install/test_installation.sh b/install/test_installation.sh new file mode 100644 index 000000000..4a301ea48 --- /dev/null +++ b/install/test_installation.sh @@ -0,0 +1,29 @@ +#!/bin/bash +tests_failed=0 + +# test validity of featurebase.service +systemd-analyze verify /etc/systemd/system/featurebase.service +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo 'featurebase.redhat.service invalid' + tests_failed=1 +fi + +# test validity of featurebase.conf +mkdir /var/log/molecula +featurebase_bin='/usr/local/bin/featurebase' +config_file='/etc/featurebase.conf' +if ! $featurebase_bin -c $config_file holder /dev/null 2>&1; then + echo 'featurebase.conf is invalid' + tests_failed=1 +fi + +# print success if both passed +if [ $tests_failed -eq 0 ]; then + echo 'featurebase.redhat.service is valid' + echo 'featurebase.conf is valid' +fi + +# tests_failed set to 0 if none of the tests failed +# otherwise set to non-zero +exit $tests_failed