From ba70bee875c9e095bf01189d9b95f358aab67d66 Mon Sep 17 00:00:00 2001 From: Bruce Baranowski <92940816+bruce-b-molecula@users.noreply.github.com> Date: Tue, 15 Nov 2022 15:55:57 -0500 Subject: [PATCH] Re-allowed concurrency for molecula-consumer-csv when not using '--auto-generate' (#2294) Concurrency was previously fully disabled due to duplication when using '--auto-generate' but testing has shown that it works correctly when not using that flag. Added the required check and updated help text --- idk/cmd/molecula-consumer-csv/main.go | 4 ++-- idk/ingest.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/idk/cmd/molecula-consumer-csv/main.go b/idk/cmd/molecula-consumer-csv/main.go index 0d1b4f37e..d22c93ea5 100644 --- a/idk/cmd/molecula-consumer-csv/main.go +++ b/idk/cmd/molecula-consumer-csv/main.go @@ -30,8 +30,8 @@ func main() { return } - if m.Concurrency != 1 { - m.Log().Infof("Concurrency is not supported for csv ingest. '--concurrency' flag will be ignored.") + if m.Concurrency != 1 && m.AutoGenerate == true { + m.Log().Infof("Concurrency is not supported for csv ingest when using '--auto-generate'. '--concurrency' flag will be ignored and concurrency will be set to 1.") m.Concurrency = 1 } diff --git a/idk/ingest.go b/idk/ingest.go index 8a463dccd..0fbef5d38 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -69,7 +69,7 @@ type Main struct { AutoGenerate bool `short:"a" help:"Automatically generate IDs."` ExternalGenerate bool `short:"" help:"Use FeatureBase's ID generation (must be set alongside auto-generate)."` IDAllocKeyPrefix string `short:"" help:"A prefix for ID allocator keys when using FeatureBase's ID generation (must be different for each concurrent ingester)."` - Concurrency int `short:"c" help:"Number of concurrent sources and indexing routines to launch. Concurrency is not supported for molecula-consumer-csv and molecula-consumer-sql"` + Concurrency int `short:"c" help:"Number of concurrent sources and indexing routines to launch. Concurrency is not supported for molecula-consumer-sql. Concurrency for molecula-consumer-csv only works when providing multiple files and does not support '--auto-generate'"` CacheLength uint64 `short:"" help:"Number of batches of ID mappings to cache."` PackBools string `short:"k" help:"If non-empty, boolean fields will be packed into two set fields—one with this name, and one with -exists."` Verbose bool `short:"v" help:"Enable verbose logging."`