bbpe: Add token LRU cache and avoid string lookups #243
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Test | |
on: [push, pull_request, workflow_call] | |
jobs: | |
test: | |
name: Run tests | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
os: [ubuntu-latest, macos-latest, windows-latest] | |
python-version: ["3.8", "3.12"] | |
steps: | |
- uses: actions/checkout@v1 | |
with: | |
submodules: true | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
cache: "pip" # caching pip dependencies | |
- name: Install requirements | |
run: | | |
python -m pip install --upgrade pip setuptools wheel build | |
python -m pip install -r requirements.txt | |
- name: Build sdist | |
run: | | |
python -m build --sdist | |
- name: Run mypy | |
run: mypy curated_tokenizers | |
- name: Delete source directory | |
run: | | |
rm -rf curated_tokenizers | |
shell: bash | |
- name: Uninstall all packages | |
run: | | |
python -m pip freeze | |
python -m pip freeze --exclude pywin32 > installed.txt | |
python -m pip uninstall -y -r installed.txt | |
- name: Install from sdist | |
run: | | |
SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) | |
python -m pip install dist/$SDIST | |
shell: bash | |
- name: Install test dependencies | |
run: python -m pip install -r requirements.txt | |
- name: Run pytest | |
run: python -m pytest --pyargs curated_tokenizers |