diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 6d5ce4f..0000000 --- a/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -exclude = - .git, - __pycache__, - build -max-complexity = 10 diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml deleted file mode 100644 index 0336013..0000000 --- a/.github/workflows/init.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Workflow runs only once when the template is first used. -# File can be safely deleted after repo is initialized. -name: Initialize repository -on: - push: - branches: - - main - -jobs: - initialize-package: - name: Initialize the package - if: ${{github.event.repository.name != 'aind-library-template'}} - runs-on: ubuntu-latest - env: - REPO_NAME: ${{ github.event.repository.name }} - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Rename package - run: | - pkg_name=$(echo "${REPO_NAME}" | tr - _) - current_description='description = "Prints messages to stdout. Simple boilerplate for libraries."' - new_description='description = "Generated from aind-library-template"' - readme_description='Template for a minimal, basic repository for an AIND library.' - new_readme_description='Generated from aind-library-template' - echo "Package Name ${pkg_name}" - mkdir src/${pkg_name} - touch src/${pkg_name}/__init__.py - echo '"""Init package"""' >> src/${pkg_name}/__init__.py - echo '__version__ = "0.0.0"' >> src/${pkg_name}/__init__.py - sed -i "s/aind_library_template/${pkg_name}/" pyproject.toml - sed -i "s/aind-library-template/${REPO_NAME}/" pyproject.toml - sed -i "s/aind_library_template/${pkg_name}/" doc_template/source/conf.py - sed -i "s/${current_description}/${new_description}/" pyproject.toml - sed -i "/pandas/d" pyproject.toml - sed -i "s/aind-library-template/${REPO_NAME}/" README.md - sed -i "s/${readme_description}/${new_readme_description}/" README.md - - name: Commit changes - uses: EndBug/add-and-commit@v9 - with: - default_author: github_actions - message: "ci: version bump [skip actions]" - add: '["pyproject.toml", "README.md", "src/*", "doc_template/source/conf.py"]' - remove: '["-r src/aind_library_template", "tests/test_message_handler.py"]' - - name: Add first tag - run: | - git tag v0.0.0 - git push origin v0.0.0 - - name: Disable workflow - run: | - gh workflow disable -R $GITHUB_REPOSITORY "${{ github.workflow }}" diff --git a/.github/workflows/tag_and_publish.yml b/.github/workflows/tag_and_publish.yml index 90419da..6383458 100644 --- a/.github/workflows/tag_and_publish.yml +++ b/.github/workflows/tag_and_publish.yml @@ -3,87 +3,77 @@ on: push: branches: - main -# Remove line 61 to enable automated semantic version bumps. -# Change line 67 from "if: false" to "if: true" to enable PyPI publishing. -# Requires that svc-aindscicomp be added as an admin to repo. jobs: - update_badges: + tag: + name: Bump version runs-on: ubuntu-latest - continue-on-error: true + outputs: + new_version: ${{ steps.output_version.outputs.new_version }} steps: - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v3 with: - ref: ${{ env.DEFAULT_BRANCH }} + ref: "main" fetch-depth: 0 token: ${{ secrets.SERVICE_TOKEN }} - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: 3.8 - - name: Install dependencies - run: | - python -m pip install -e .[dev] --no-cache-dir - - name: Get Python version and Update README.md - run: | - python_version=$(grep "requires-python" pyproject.toml | grep -o ">=[^\"]*") - python_badge=$(grep -o 'python-[^)]*' README.md) - new_python_badge="python-$python_version-blue?logo=python" - sed -i "s/$python_badge/$new_python_badge/g" README.md - - name: Get interrogate values and Update README.md - run: | - interrogate_val=$(interrogate . | grep -o 'actual: [0-9]*\.[0-9]*' | awk '{print $2}') - interrogate_badge=$(grep -o 'interrogate-[^)]*' README.md) - if (( $(echo "$interrogate_val >= 90.00" | bc -l) )); then - new_interrogate_badge="interrogate-$interrogate_val%25-brightgreen" - elif (( $(echo "$interrogate_val < 80.00" | bc -l) )); then - new_interrogate_badge="interrogate-$interrogate_val%25-red" - else - new_interrogate_badge="interrogate-$interrogate_val%25-yellow" - fi - sed -i "s/$interrogate_badge/$new_interrogate_badge/g" README.md - - name: Get Coverage values and Update README.md - run: | - coverage run -m unittest discover - coverage_val=$(coverage report | grep "^TOTAL" | grep -o '[0-9]\+%' | grep -o '[0-9]\+') - coverage_badge=$(grep -o "coverage-[^?]*" README.md) - if (( $(echo "$coverage_val >= 90.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-brightgreen" - elif (( $(echo "$coverage_val < 80.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-red" - else - new_coverage_badge="coverage-$coverage_val%25-yellow" - fi - sed -i "s/$coverage_badge/$new_coverage_badge/g" README.md - - name: Commit changes - uses: EndBug/add-and-commit@v9 + - name: Compute new version number + id: tag_version + uses: mathieudutour/github-tag-action@v6.1 with: - default_author: github_actions - message: "ci: update badges [skip actions]" - add: '["README.md"]' - tag: - needs: update_badges - if: ${{github.event.repository.name == 'aind-library-template'}} - uses: AllenNeuralDynamics/aind-github-actions/.github/workflows/tag.yml@main - secrets: - SERVICE_TOKEN: ${{ secrets.SERVICE_TOKEN }} + github_token: ${{ secrets.SERVICE_TOKEN }} + release_branches: "main" + default_bump: patch + dry_run: false publish: - needs: tag - if: false runs-on: ubuntu-latest + env: + GO_VERSION: '1.14.12' + GO_OS: 'linux' + GO_ARCH: 'amd64' + SINGULARITY_VERSION: '3.7.0' + needs: tag steps: - uses: actions/checkout@v3 - name: Pull latest changes run: git pull origin main - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - name: Install dependencies run: | - pip install --upgrade setuptools wheel twine build - python -m build - twine check dist/* - - name: Publish on PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.AIND_PYPI_TOKEN }} + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + wget \ + pkg-config \ + procps + - name: Install GO + run: | + wget https://dl.google.com/go/go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz + sudo tar -C /usr/local -xzvf go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz + sudo rm go$GO_VERSION.$GO_OS-$GO_ARCH.tar.gz + echo 'export PATH=$PATH:/usr/local/go/bin' | sudo tee -a /etc/profile + - name: Install Singularity + run: | + cd .. + wget https://github.com/hpcng/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz + sudo tar -xzf singularity-${SINGULARITY_VERSION}.tar.gz + sudo rm singularity-${SINGULARITY_VERSION}.tar.gz + echo "Finished installing binaries" + export PATH=$PATH:/usr/local/go/bin + cd singularity + sudo ./mconfig --without-suid + sudo make -C ./builddir + sudo make -C ./builddir install + - name: Build sif container + run: | + ls / + ls + mkdir build + sudo singularity build build/container.sif scripts/singularity_build.def + - name: Login and Deploy Container + run: | + echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ github.actor }} --password-stdin oras://ghcr.io + singularity push build/container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:${{ needs.tag.outputs.new_version }} + singularity push build/container.sif oras://ghcr.io/${GITHUB_REPOSITORY}:latest diff --git a/.github/workflows/test_and_lint.yml b/.github/workflows/test_and_lint.yml deleted file mode 100644 index c8d832d..0000000 --- a/.github/workflows/test_and_lint.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Lint and run tests - -on: - pull_request: - branches: - - main - -jobs: - ci: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ '3.8', '3.9', '3.10' ] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install -e .[dev] - - name: Run linter checks - run: flake8 . && interrogate --verbose . - - name: Run tests and coverage - run: coverage run -m unittest discover && coverage report diff --git a/README.md b/README.md index f38fd32..b3a3316 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,9 @@ # aind-data-upload +Repository to manage uploading data to the cloud + [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE) -![Code Style](https://img.shields.io/badge/code%20style-black-black) [![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release) -![Interrogate](https://img.shields.io/badge/interrogate-100.0%25-brightgreen) -![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen?logo=codecov) -![Python](https://img.shields.io/badge/python->=3.7-blue?logo=python) - - ## Usage - - To use this template, click the green `Use this template` button and `Create new repository`. - - After github initially creates the new repository, please wait an extra minute for the initialization scripts to finish organizing the repo. - - To enable the automatic semantic version increments: in the repository go to `Settings` and `Collaborators and teams`. Click the green `Add people` button. Add `svc-aindscicomp` as an admin. Modify the file in `.github/workflows/tag_and_publish.yml` and remove the if statement in line 10. The semantic version will now be incremented every time a code is committed into the main branch. - - To publish to PyPI, enable semantic versioning and uncomment the publish block in `.github/workflows/tag_and_publish.yml`. The code will now be published to PyPI every time the code is committed into the main branch. - - The `.github/workflows/test_and_lint.yml` file will run automated tests and style checks every time a Pull Request is opened. If the checks are undesired, the `test_and_lint.yml` can be deleted. The strictness of the code coverage level, etc., can be modified by altering the configurations in the `pyproject.toml` file and the `.flake8` file. - -## Installation -To use the software, in the root directory, run -```bash -pip install -e . -``` - -To develop the code, run -```bash -pip install -e .[dev] -``` - -## Contributing - -### Linters and testing - -There are several libraries used to run linters, check documentation, and run tests. - -- Please test your changes using the **coverage** library, which will run the tests and log a coverage report: - -```bash -coverage run -m unittest discover && coverage report -``` - -- Use **interrogate** to check that modules, methods, etc. have been documented thoroughly: - -```bash -interrogate . -``` - -- Use **flake8** to check that code is up to standards (no unused imports, etc.): -```bash -flake8 . -``` - -- Use **black** to automatically format the code into PEP standards: -```bash -black . -``` - -- Use **isort** to automatically sort import statements: -```bash -isort . -``` - -### Pull requests - -For internal members, please create a branch. For external members, please fork the repository and open a pull request from the fork. We'll primarily use [Angular](https://github.com/angular/angular/blob/main/CONTRIBUTING.md#commit) style for commit messages. Roughly, they should follow the pattern: -```text -(): -``` - -where scope (optional) describes the packages affected by the code changes and type (mandatory) is one of: - -- **build**: Changes that affect build tools or external dependencies (example scopes: pyproject.toml, setup.py) -- **ci**: Changes to our CI configuration files and scripts (examples: .github/workflows/ci.yml) -- **docs**: Documentation only changes -- **feat**: A new feature -- **fix**: A bugfix -- **perf**: A code change that improves performance -- **refactor**: A code change that neither fixes a bug nor adds a feature -- **test**: Adding missing tests or correcting existing tests - -### Semantic Release - -The table below, from [semantic release](https://github.com/semantic-release/semantic-release), shows which commit message gets you which release type when `semantic-release` runs (using the default configuration): - -| Commit message | Release type | -| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| `fix(pencil): stop graphite breaking when too much pressure applied` | ~~Patch~~ Fix Release, Default release | -| `feat(pencil): add 'graphiteWidth' option` | ~~Minor~~ Feature Release | -| `perf(pencil): remove graphiteWidth option`

`BREAKING CHANGE: The graphiteWidth option has been removed.`
`The default graphite width of 10mm is always used for performance reasons.` | ~~Major~~ Breaking Release
(Note that the `BREAKING CHANGE: ` token must be in the footer of the commit) | - -### Documentation -To generate the rst files source files for documentation, run -```bash -sphinx-apidoc -o doc_template/source/ src -``` -Then to create the documentation HTML files, run -```bash -sphinx-build -b html doc_template/source/ doc_template/build/html -``` -More info on sphinx installation can be found [here](https://www.sphinx-doc.org/en/master/usage/installation.html). +We can add additional python if needed. For now, this will publish a singularity container with `awscli` installed. Most jobs will just use the sync command. diff --git a/doc_template/Makefile b/doc_template/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/doc_template/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc_template/make.bat b/doc_template/make.bat deleted file mode 100644 index dc1312a..0000000 --- a/doc_template/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/doc_template/source/_static/dark-logo.svg b/doc_template/source/_static/dark-logo.svg deleted file mode 100644 index dcc68fb..0000000 --- a/doc_template/source/_static/dark-logo.svg +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc_template/source/_static/favicon.ico b/doc_template/source/_static/favicon.ico deleted file mode 100644 index 4cec150..0000000 Binary files a/doc_template/source/_static/favicon.ico and /dev/null differ diff --git a/doc_template/source/_static/light-logo.svg b/doc_template/source/_static/light-logo.svg deleted file mode 100644 index b20cb67..0000000 --- a/doc_template/source/_static/light-logo.svg +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/doc_template/source/conf.py b/doc_template/source/conf.py deleted file mode 100644 index e39e111..0000000 --- a/doc_template/source/conf.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Configuration file for the Sphinx documentation builder.""" -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path Setup -------------------------------------------------------------- -from os.path import dirname, abspath -from pathlib import Path -from datetime import date -from aind_data_upload import __version__ as package_version - -INSTITUTE_NAME = "Allen Institute for Neural Dynamics" - -current_year = date.today().year - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = Path(dirname(dirname(dirname(abspath(__file__))))).name -copyright = f"{current_year}, {INSTITUTE_NAME}" -author = INSTITUTE_NAME -release = package_version - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = [ - "sphinx.ext.duration", - "sphinx.ext.doctest", - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", -] -templates_path = ["_templates"] -exclude_patterns = [] - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "furo" -html_static_path = ["_static"] -html_favicon = "_static/favicon.ico" -html_theme_options = { - "light_logo": "light-logo.svg", - "dark_logo": "dark-logo.svg", -} - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -html_show_sphinx = False - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -html_show_copyright = False diff --git a/doc_template/source/index.rst b/doc_template/source/index.rst deleted file mode 100644 index 07adcad..0000000 --- a/doc_template/source/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. Doc Template documentation master file, created by - sphinx-quickstart on Wed Aug 17 15:36:32 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - - -Welcome to this repository's documentation! -=========================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - modules - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index af07bef..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,79 +0,0 @@ -[build-system] -requires = ["setuptools", "setuptools-scm"] -build-backend = "setuptools.build_meta" - -[project] -name = "aind-data-upload" -description = "Generated from aind-library-template" -license = {text = "MIT"} -requires-python = ">=3.7" -authors = [ - {name = "Allen Institute for Neural Dynamics"} -] -classifiers = [ - "Programming Language :: Python :: 3" -] -readme = "README.md" -dynamic = ["version"] - -dependencies = [ -] - -[project.optional-dependencies] -dev = [ - 'black', - 'coverage', - 'flake8', - 'interrogate', - 'isort', - 'Sphinx', - 'furo' -] - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.dynamic] -version = {attr = "aind_data_upload.__version__"} - -[tool.black] -line-length = 79 -target_version = ['py36'] -exclude = ''' - -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | build - | dist - )/ - | .gitignore -) -''' - -[tool.coverage.run] -omit = ["*__init__*"] -source = ["aind_data_upload", "tests"] - -[tool.coverage.report] -exclude_lines = [ - "if __name__ == .__main__.:", - "from", - "import", - "pragma: no cover" -] -fail_under = 100 - -[tool.isort] -line_length = 79 -profile = "black" - -[tool.interrogate] -exclude = ["setup.py", "docs", "build"] -fail-under = 100 diff --git a/scripts/singularity_build.def b/scripts/singularity_build.def new file mode 100644 index 0000000..845f836 --- /dev/null +++ b/scripts/singularity_build.def @@ -0,0 +1,8 @@ +Bootstrap: docker +From: python:3.10-bullseye + +%post + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + rm awscliv2.zip + ./aws/install diff --git a/setup.py b/setup.py deleted file mode 100644 index 7f1a176..0000000 --- a/setup.py +++ /dev/null @@ -1,4 +0,0 @@ -from setuptools import setup - -if __name__ == "__main__": - setup() diff --git a/src/aind_data_upload/__init__.py b/src/aind_data_upload/__init__.py deleted file mode 100644 index d0a8547..0000000 --- a/src/aind_data_upload/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Init package""" -__version__ = "0.0.0" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 816e430..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Testing library""" diff --git a/tests/test_example.py b/tests/test_example.py deleted file mode 100644 index 06e9e0d..0000000 --- a/tests/test_example.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Example test template.""" - -import unittest - - -class ExampleTest(unittest.TestCase): - """Example Test Class""" - - def test_assert_example(self): - """Example of how to test the truth of a statement.""" - - self.assertTrue(1 == 1) - - -if __name__ == "__main__": - unittest.main()