mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-13 08:01:02 +00:00
Merge pull request #1646 from tgruben/ci-backup
[CORE-629] CI test for backup restore
This commit is contained in:
tgruben
GitHub
commit
69ef605f93
6 changed files with 188 additions and 0 deletions
|
|
@ -151,6 +151,16 @@ jobs:
|
|||
- run:
|
||||
command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
|
||||
no_output_timeout: 30m
|
||||
test-backup-restore:
|
||||
executor:
|
||||
name: golang
|
||||
steps:
|
||||
- checkout-plus
|
||||
- skip-if-root-unchanged
|
||||
- setup_remote_docker
|
||||
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
|
||||
- run: make backuptests-build
|
||||
- run: make backuptests
|
||||
cluster-tests:
|
||||
executor:
|
||||
name: golang
|
||||
|
|
@ -274,6 +284,10 @@ workflows:
|
|||
context: molecula
|
||||
requires:
|
||||
- setup
|
||||
- test-backup-restore:
|
||||
context: molecula
|
||||
requires:
|
||||
- setup
|
||||
- cluster-tests:
|
||||
context: molecula
|
||||
requires:
|
||||
|
|
|
|||
38
Dockerfile.pilosa
Normal file
38
Dockerfile.pilosa
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
ARG GO_VERSION=latest
|
||||
|
||||
######################
|
||||
### Pilosa builder ###
|
||||
######################
|
||||
|
||||
FROM golang:${GO_VERSION} as pilosa-builder
|
||||
ARG MAKE_FLAGS
|
||||
WORKDIR /pilosa
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
|
||||
|
||||
#####################
|
||||
### Pilosa runner ###
|
||||
#####################
|
||||
|
||||
FROM alpine:3.13.2 as runner
|
||||
|
||||
LABEL maintainer "dev@molecula.com"
|
||||
|
||||
RUN apk add --no-cache curl jq
|
||||
|
||||
COPY --from=pilosa-builder /pilosa/build/pilosa /
|
||||
|
||||
COPY LICENSE /LICENSE
|
||||
COPY NOTICE /NOTICE
|
||||
|
||||
EXPOSE 10101
|
||||
VOLUME /data
|
||||
|
||||
ENV PILOSA_DATA_DIR /data
|
||||
ENV PILOSA_BIND 0.0.0.0:10101
|
||||
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
|
||||
|
||||
ENTRYPOINT ["/pilosa"]
|
||||
CMD ["server"]
|
||||
22
Dockerfile.runner
Normal file
22
Dockerfile.runner
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
ARG GO_VERSION=latest
|
||||
|
||||
######################
|
||||
### Pilosa builder ###
|
||||
######################
|
||||
|
||||
FROM golang:${GO_VERSION} as pilosa-builder
|
||||
ARG MAKE_FLAGS
|
||||
WORKDIR /pilosa
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
|
||||
|
||||
FROM moleculacorp/idk as idk
|
||||
RUN apk add --no-cache bash curl jq
|
||||
LABEL maintainer "dev@molecula.com"
|
||||
|
||||
|
||||
COPY --from=pilosa-builder /pilosa/build/pilosa /
|
||||
COPY testBackupRestore.sh /
|
||||
CMD ["bash","/testBackupRestore.sh"]
|
||||
9
Makefile
9
Makefile
|
|
@ -141,6 +141,15 @@ clustertests: vendor
|
|||
clustertests-build: vendor
|
||||
docker-compose -f $(DOCKER_COMPOSE) down -v
|
||||
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
|
||||
# Test Cluster backup and restore
|
||||
backuptests-build: vendor
|
||||
docker-compose -f docker-compose-3.yml down
|
||||
docker-compose -f docker-compose-3.yml build
|
||||
|
||||
backuptests: vendor
|
||||
docker-compose -f docker-compose-3.yml down -v
|
||||
docker-compose -f docker-compose-3.yml up --exit-code-from=client1 --abort-on-container-exit
|
||||
|
||||
|
||||
# Install Pilosa
|
||||
install:
|
||||
|
|
|
|||
78
docker-compose-3.yml
Normal file
78
docker-compose-3.yml
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
version: "3"
|
||||
services:
|
||||
pilosa0:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa0:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa0:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa0
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa0:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa0:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa0
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
healthcheck:
|
||||
test: x=$$(curl -s localhost:10101/status | jq -r ".state") && [[ "$$x" == "NORMAL" ]] || $$(exit 1)
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
pilosa1:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa1:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa1:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa1
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa1:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa1:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa1
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
pilosa2:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa2:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa2:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa2
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa2:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa2:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa2
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
client1:
|
||||
image: tgruben/bash
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.runner
|
||||
environment:
|
||||
- GO111MODULE=on
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
depends_on:
|
||||
- pilosa0
|
||||
|
||||
volumes:
|
||||
data:
|
||||
27
testBackupRestore.sh
Executable file
27
testBackupRestore.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
declare STATUS="NORMAL"
|
||||
declare TIMEOUT=30
|
||||
sleep 4
|
||||
STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \
|
||||
'while [[ ${STATUS_RECEIVED} != ${STATUS} ]];\
|
||||
do STATUS_RECEIVED=$(curl --connect-timeout 1 -s pilosa0:10101/status | jq -r ".state") && \
|
||||
echo "received status: $STATUS_RECEIVED" && \
|
||||
sleep 1;\
|
||||
done;'
|
||||
echo "NOW DO STUFF"
|
||||
datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
|
||||
before=$(/pilosa chksum --host pilosa0:10101)
|
||||
/pilosa backup -o backupdir --host pilosa0:10101
|
||||
curl -X DELETE -s pilosa0:10101/index/sink
|
||||
/pilosa restore -s backupdir --host pilosa0:10101
|
||||
after=$(/pilosa chksum --host pilosa0:10101)
|
||||
if [ "$before" = "$after" ]; then
|
||||
echo "PASS"
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL"
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Add table
Reference in a new issue