mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-17 04:37:00 +00:00
514ca643a1
* feat: add Qwen token counting utility with fallback * fix: preserve Qwen tokenizer text prefixes and isolate loading * fix: isolate injected Qwen tokenizers and refresh local sources * perf: avoid unnecessary Qwen tokenizer initialization and locking --------- Co-authored-by: jinli.yl <jinli.yl@alibaba-inc.com>
64 lines
2 KiB
YAML
64 lines
2 KiB
YAML
name: CI / Python tests
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
unit-tests:
|
|
name: Unit Tests - py${{ matrix.python-version }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 90
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
cache: 'pip'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip setuptools wheel
|
|
pip install -e reme_studio -e ".[dev,core,classifier-jev,qwen-tokenizer]"
|
|
pip install --no-deps -e plugins/auto-fin
|
|
pip install -e plugins/daily_paper
|
|
pip install -e plugins/lme -e plugins/beam
|
|
pip install coverage
|
|
|
|
- name: Run unit tests
|
|
run: |
|
|
python -c "import tokenizers; print('tokenizers', tokenizers.__version__)"
|
|
coverage run -m pytest tests/unit plugins/auto-fin plugins/daily_paper plugins/lme plugins/beam \
|
|
-v \
|
|
--tb=long \
|
|
-s \
|
|
--log-cli-level=WARNING
|
|
|
|
- name: Generate coverage report
|
|
run: coverage report -m
|
|
|
|
- name: Verify minimum tokenizer version
|
|
if: matrix.python-version == '3.11'
|
|
run: |
|
|
python -m pip install "tokenizers==0.21.0"
|
|
python -c "import tokenizers; print('tokenizers', tokenizers.__version__)"
|
|
python -m pytest tests/unit/test_qwen_token_tools.py tests/unit/test_package_versions.py -q
|