From a551143367ab204a989794b7e956b1826a646213 Mon Sep 17 00:00:00 2001 From: Judah Jacobson Date: Sun, 20 May 2018 11:05:43 -0700 Subject: [PATCH] Fix #185: Support and use autogen-modules. This makes it possible to generate tarballs without generated modules (which would be regenerated anyway when the package is build). See tensorflow/haskell#180 for an example of the issues that causes. Unfortunately, this required us stopping using `hpack` in our packages that generate proto files. `hpack`, when you use the `autogen-modules` field, `cabal-version: >= 2.0` when the file I mitigated the situation a little by changing the Cabal test script to only try to `sdist` packages that we're planning to release. --- .travis.yml | 2 +- docs/releasing.md | 22 +---- proto-lens-combinators/package.yaml | 50 ------------ .../proto-lens-combinators.cabal | 65 +++++++++++++++ proto-lens-discrimination/package.yaml | 56 ------------- .../proto-lens-discrimination.cabal | 81 +++++++++++++++++++ proto-lens-protobuf-types/package.yaml | 44 ---------- .../proto-lens-protobuf-types.cabal | 61 ++++++++++++++ proto-lens-protoc/src/Data/ProtoLens/Setup.hs | 5 ++ proto-lens-tests-dep/package.yaml | 34 -------- .../proto-lens-tests-dep.cabal | 39 +++++++++ release.sh | 21 ----- travis-cabal.sh | 65 ++++++++++----- 13 files changed, 298 insertions(+), 247 deletions(-) delete mode 100644 proto-lens-combinators/package.yaml create mode 100644 proto-lens-combinators/proto-lens-combinators.cabal delete mode 100644 proto-lens-discrimination/package.yaml create mode 100644 proto-lens-discrimination/proto-lens-discrimination.cabal delete mode 100644 proto-lens-protobuf-types/package.yaml create mode 100644 proto-lens-protobuf-types/proto-lens-protobuf-types.cabal delete mode 100644 proto-lens-tests-dep/package.yaml create mode 100644 proto-lens-tests-dep/proto-lens-tests-dep.cabal delete mode 100755 release.sh diff --git a/.travis.yml b/.travis.yml index a4c92349..83c9ecf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ before_install: cabal) export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.local/bin:$HOME/.cabal/bin:$PATH;; esac - - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.6.1/stack-1.6.1-linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C $HOME/.local/bin '*/stack' + - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.7.1/stack-1.7.1-linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C $HOME/.local/bin '*/stack' - curl -L https://github.com/google/protobuf/releases/download/v3.0.0-beta-2/protoc-3.0.0-beta-2-linux-x86_64.zip > protoc-release.zip - unzip -p protoc-release.zip protoc > $HOME/.local/bin/protoc - chmod a+x $HOME/.local/bin/protoc diff --git a/docs/releasing.md b/docs/releasing.md index a2ebe9e1..af0cf7a1 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -17,26 +17,8 @@ since they are intended only for use within this repository: - Make sure you are on a commit that has been merged upstream to `google/proto-lens`. ## Steps -1. For each package *without* autogenerated proto modules, simply run (from the root of the repo): +First, run `stack build` at the top level. Then, for each package that you want to upload, run stack upload {name-of-package} - Then verify that they've been uploaded successfully by visiting `hackage.org` in a web browser. - -2. For packages *with* autogenerated proto-modules (currently: -`proto-lens-protobuf-types` and `proto-lens-combinators`): these are a little -harder to upload due to #185. We have a script for running `sdist`: - - ./release.sh - - Then, for each package that should be uploaded: - - cabal upload {package}/dist/{path}.tar.gz - -3. Finally, tag each uploaded package with its version and push it back to upstream. For example: - - git tag -a proto-lens-v0.3.0.0 -m "Release proto-lens v0.3.0.0" - ... - git remote add upstream git@github.com:google/proto-lens.git - git push upstream --follow-tags # See: https://stackoverflow.com/a/26438076 - +Then verify that they've been uploaded successfully by visiting `hackage.org` in a web browser. diff --git a/proto-lens-combinators/package.yaml b/proto-lens-combinators/package.yaml deleted file mode 100644 index caecf3a8..00000000 --- a/proto-lens-combinators/package.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: proto-lens-combinators -version: '0.1.0.10' -synopsis: Utilities functions to proto-lens. -description: Useful things for working with protos. -category: Data -author: Zie Weaver -maintainer: zearen+protolens@google.com -copyright: Google Inc. -license: BSD3 -github: google/proto-lens/proto-lens-combinators -extra-source-files: - - tests/combinators.proto - - Changelog.md - -custom-setup: - dependencies: - - base >= 4.8 && < 4.12 - - Cabal - - proto-lens-protoc == 0.3.* - -dependencies: - - base >= 4.8 && < 4.12 - - proto-lens-protoc == 0.3.* - - lens-family == 1.2.* - -library: - source-dirs: src - exposed-modules: - - Data.ProtoLens.Combinators - dependencies: - - data-default-class >= 0.0 && < 0.2 - - transformers >= 0.4 && < 0.6 - -tests: - combinators_test: - main: combinators_test.hs - source-dirs: tests - dependencies: - - HUnit - - base - - lens-family - - lens-family-core - - proto-lens - - proto-lens-combinators - - proto-lens-protoc - - test-framework - - test-framework-hunit - other-modules: - Proto.Combinators - Proto.Combinators_Fields diff --git a/proto-lens-combinators/proto-lens-combinators.cabal b/proto-lens-combinators/proto-lens-combinators.cabal new file mode 100644 index 00000000..f14fd4ae --- /dev/null +++ b/proto-lens-combinators/proto-lens-combinators.cabal @@ -0,0 +1,65 @@ +name: proto-lens-combinators +version: 0.1.0.10 +synopsis: Utilities functions to proto-lens. +description: Useful things for working with protos. +category: Data +homepage: https://github.com/google/proto-lens#readme +bug-reports: https://github.com/google/proto-lens/issues +author: Zie Weaver +maintainer: zearen+protolens@google.com +copyright: Google Inc. +license: BSD3 +license-file: LICENSE +build-type: Custom +cabal-version: >= 1.24 +extra-source-files: + Changelog.md + tests/combinators.proto + +source-repository head + type: git + location: https://github.com/google/proto-lens + subdir: proto-lens-combinators + +custom-setup + setup-depends: + Cabal + , base >=4.8 && <4.12 + , proto-lens-protoc ==0.3.* + +library + exposed-modules: + Data.ProtoLens.Combinators + other-modules: + Paths_proto_lens_combinators + hs-source-dirs: + src + build-depends: + base >=4.8 && <4.12 + , data-default-class >=0.0 && <0.2 + , lens-family ==1.2.* + , proto-lens-protoc ==0.3.* + , transformers >=0.4 && <0.6 + default-language: Haskell2010 + +test-suite combinators_test + type: exitcode-stdio-1.0 + main-is: combinators_test.hs + other-modules: + Paths_proto_lens_combinators + Proto.Combinators Proto.Combinators_Fields + autogen-modules: + Proto.Combinators Proto.Combinators_Fields + hs-source-dirs: + tests + build-depends: + HUnit + , base + , lens-family + , lens-family-core + , proto-lens + , proto-lens-combinators + , proto-lens-protoc + , test-framework + , test-framework-hunit + default-language: Haskell2010 diff --git a/proto-lens-discrimination/package.yaml b/proto-lens-discrimination/package.yaml deleted file mode 100644 index b365c01f..00000000 --- a/proto-lens-discrimination/package.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: proto-lens-discrimination -version: '0.1.1.0' -synopsis: Support for using proto-lens types with http://hackage.haskell.org/package/discrimination. -description: > - Reflection-based implementations of Sort for proto-lens' - generated types. -category: Data -author: Andrew Pritchard -maintainer: awpr+protolens@google.com -copyright: Google Inc. -license: BSD3 -github: google/proto-lens/proto-lens-discrimination -extra-source-files: - - tests/*.proto - -custom-setup: - dependencies: - - base >= 4.8 && < 4.12 - - Cabal - - proto-lens-protoc == 0.3.* - -dependencies: - - base >= 4.8 && < 4.12 - - data-default >= 0.5 && < 0.8 - - bytestring == 0.10.* - - contravariant >= 1.3 && < 1.5 - - containers >= 0.5 && < 0.6 - - discrimination >= 0.3 && < 0.4 - - discrimination-ieee754 == 0.1.* - - lens-family == 1.2.* - - proto-lens >= 0.3 && < 0.4 - - proto-lens-protoc >= 0.3 && < 0.4 - - text == 1.2.* - -library: - source-dirs: src - exposed-modules: - - Data.ProtoLens.Sort - - Data.ProtoLens.Group - - Data.ProtoLens.Discrimination - -tests: - - discrimination_test: - main: disc_test.hs - source-dirs: tests - dependencies: - - HUnit >= 1.3 && < 1.7 - - proto-lens-arbitrary >= 0.1 && < 0.2 - - proto-lens-discrimination - - test-framework == 0.8.* - - test-framework-hunit == 0.3.* - - test-framework-quickcheck2 == 0.3.* - other-modules: - - Proto.Enum - - Proto.Enum_Fields diff --git a/proto-lens-discrimination/proto-lens-discrimination.cabal b/proto-lens-discrimination/proto-lens-discrimination.cabal new file mode 100644 index 00000000..5a5b3e0d --- /dev/null +++ b/proto-lens-discrimination/proto-lens-discrimination.cabal @@ -0,0 +1,81 @@ +name: proto-lens-discrimination +version: 0.1.1.0 +synopsis: Support for using proto-lens types with http://hackage.haskell.org/package/discrimination. +description: Reflection-based implementations of Sort for proto-lens' generated types. +category: Data +homepage: https://github.com/google/proto-lens#readme +bug-reports: https://github.com/google/proto-lens/issues +author: Andrew Pritchard +maintainer: awpr+protolens@google.com +copyright: Google Inc. +license: BSD3 +license-file: LICENSE +build-type: Custom +cabal-version: >= 1.24 +extra-source-files: + tests/enum.proto + +source-repository head + type: git + location: https://github.com/google/proto-lens + subdir: proto-lens-discrimination + +custom-setup + setup-depends: + Cabal + , base >=4.8 && <4.12 + , proto-lens-protoc ==0.3.* + +library + exposed-modules: + Data.ProtoLens.Sort + Data.ProtoLens.Group + Data.ProtoLens.Discrimination + other-modules: + Paths_proto_lens_discrimination + hs-source-dirs: + src + build-depends: + base >=4.8 && <4.12 + , bytestring ==0.10.* + , containers >=0.5 && <0.6 + , contravariant >=1.3 && <1.5 + , data-default >=0.5 && <0.8 + , discrimination >=0.3 && <0.4 + , discrimination-ieee754 ==0.1.* + , lens-family ==1.2.* + , proto-lens >=0.3 && <0.4 + , proto-lens-protoc >=0.3 && <0.4 + , text ==1.2.* + default-language: Haskell2010 + +test-suite discrimination_test + type: exitcode-stdio-1.0 + main-is: disc_test.hs + other-modules: + Proto.Enum + Proto.Enum_Fields + autogen-modules: + Proto.Enum + Proto.Enum_Fields + hs-source-dirs: + tests + build-depends: + HUnit >=1.3 && <1.7 + , base >=4.8 && <4.12 + , bytestring ==0.10.* + , containers >=0.5 && <0.6 + , contravariant >=1.3 && <1.5 + , data-default >=0.5 && <0.8 + , discrimination >=0.3 && <0.4 + , discrimination-ieee754 ==0.1.* + , lens-family ==1.2.* + , proto-lens >=0.3 && <0.4 + , proto-lens-arbitrary >=0.1 && <0.2 + , proto-lens-discrimination + , proto-lens-protoc >=0.3 && <0.4 + , test-framework ==0.8.* + , test-framework-hunit ==0.3.* + , test-framework-quickcheck2 ==0.3.* + , text ==1.2.* + default-language: Haskell2010 diff --git a/proto-lens-protobuf-types/package.yaml b/proto-lens-protobuf-types/package.yaml deleted file mode 100644 index 367e03ea..00000000 --- a/proto-lens-protobuf-types/package.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: proto-lens-protobuf-types -version: '0.3.0.1' -synopsis: Basic protocol buffer message types. -description: > - This package provides bindings standard protocol message types, - for use with the proto-lens library. -category: Data -author: Judah Jacobson -maintainer: proto-lens@googlegroups.com -copyright: Google Inc. -license: BSD3 -github: google/proto-lens/proto-lens-protobuf-types -extra-source-files: - - Changelog.md - - proto-src/google/protobuf/any.proto - - proto-src/google/protobuf/duration.proto - - proto-src/google/protobuf/wrappers.proto - - proto-src/google/protobuf/timestamp.proto - -custom-setup: - dependencies: - - base >= 4.8 && < 4.12 - - Cabal - - proto-lens-protoc == 0.3.* - -dependencies: - - base >= 4.8 && < 4.12 - - lens-family - - proto-lens == 0.3.* - - proto-lens-protoc == 0.3.* - - text == 1.2.* - -library: - source-dirs: src - exposed-modules: - - Data.ProtoLens.Any - - Proto.Google.Protobuf.Any - - Proto.Google.Protobuf.Any_Fields - - Proto.Google.Protobuf.Duration - - Proto.Google.Protobuf.Duration_Fields - - Proto.Google.Protobuf.Wrappers - - Proto.Google.Protobuf.Wrappers_Fields - - Proto.Google.Protobuf.Timestamp - - Proto.Google.Protobuf.Timestamp_Fields diff --git a/proto-lens-protobuf-types/proto-lens-protobuf-types.cabal b/proto-lens-protobuf-types/proto-lens-protobuf-types.cabal new file mode 100644 index 00000000..7ce66891 --- /dev/null +++ b/proto-lens-protobuf-types/proto-lens-protobuf-types.cabal @@ -0,0 +1,61 @@ +name: proto-lens-protobuf-types +version: 0.3.0.1 +synopsis: Basic protocol buffer message types. +description: This package provides bindings standard protocol message types, for use with the proto-lens library. +category: Data +homepage: https://github.com/google/proto-lens#readme +bug-reports: https://github.com/google/proto-lens/issues +author: Judah Jacobson +maintainer: proto-lens@googlegroups.com +copyright: Google Inc. +license: BSD3 +license-file: LICENSE +build-type: Custom +cabal-version: >= 1.24 +extra-source-files: + Changelog.md + proto-src/google/protobuf/any.proto + proto-src/google/protobuf/duration.proto + proto-src/google/protobuf/timestamp.proto + proto-src/google/protobuf/wrappers.proto + +source-repository head + type: git + location: https://github.com/google/proto-lens + subdir: proto-lens-protobuf-types + +custom-setup + setup-depends: + Cabal + , base >=4.8 && <4.12 + , proto-lens-protoc ==0.3.* + +library + exposed-modules: + Data.ProtoLens.Any + Proto.Google.Protobuf.Any + Proto.Google.Protobuf.Any_Fields + Proto.Google.Protobuf.Duration + Proto.Google.Protobuf.Duration_Fields + Proto.Google.Protobuf.Wrappers + Proto.Google.Protobuf.Wrappers_Fields + Proto.Google.Protobuf.Timestamp + Proto.Google.Protobuf.Timestamp_Fields + autogen-modules: + Proto.Google.Protobuf.Any + Proto.Google.Protobuf.Any_Fields + Proto.Google.Protobuf.Duration + Proto.Google.Protobuf.Duration_Fields + Proto.Google.Protobuf.Wrappers + Proto.Google.Protobuf.Wrappers_Fields + Proto.Google.Protobuf.Timestamp + Proto.Google.Protobuf.Timestamp_Fields + hs-source-dirs: + src + build-depends: + base >=4.8 && <4.12 + , lens-family + , proto-lens ==0.3.* + , proto-lens-protoc ==0.3.* + , text ==1.2.* + default-language: Haskell2010 diff --git a/proto-lens-protoc/src/Data/ProtoLens/Setup.hs b/proto-lens-protoc/src/Data/ProtoLens/Setup.hs index cb168e72..d71febb0 100644 --- a/proto-lens-protoc/src/Data/ProtoLens/Setup.hs +++ b/proto-lens-protoc/src/Data/ProtoLens/Setup.hs @@ -185,11 +185,16 @@ generatingSpecificProtos root getProtos hooks = hooks { buildHook = \p l h f -> generate l >> buildHook hooks p l h f , haddockHook = \p l h f -> generate l >> haddockHook hooks p l h f , replHook = \p l h f args -> generate l >> replHook hooks p l h f args +#if !MIN_VERSION_Cabal(2,0,0) + -- Older versions of Cabal don't support the autogen-modules field. + -- Work around it by manually generating the modules and putting them + -- in a place where `cabal sdist` will pick them up. , sDistHook = \p maybe_l h f -> case maybe_l of Nothing -> error "Can't run protoc; run 'cabal configure' first." Just l -> do generate l sDistHook hooks (fudgePackageDesc l p) maybe_l h f +#endif , postCopy = \a flags pkg lbi -> do let verb = fromFlag $ copyVerbosity flags let destDir = datadir (absoluteInstallDirs pkg lbi diff --git a/proto-lens-tests-dep/package.yaml b/proto-lens-tests-dep/package.yaml deleted file mode 100644 index e7057f31..00000000 --- a/proto-lens-tests-dep/package.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: proto-lens-tests-dep -version: '0.1.0.1' -synopsis: Test package dependency. -description: > - This package exports a .proto file which is used by proto-lens-tests - to test interpackage dependencies. - . - NOTE: The test needs a new package with its own .proto files, - rather than using the ones in proto-lens-protobuf-types - (google/protobuf/*.proto), since on some systems protoc has special logic - and can resolve imports of those files automatically. -category: Data -author: Judah Jacobson -maintainer: proto-lens@googlegroups.com -copyright: Google Inc. -license: BSD3 -github: google/proto-lens/proto-lens-tests-dep -extra-source-files: - - protos/test-dep/foo.proto - -custom-setup: - dependencies: - - base - - Cabal - - proto-lens-protoc - -dependencies: - - proto-lens-protoc - - base - -library: - exposed-modules: - - Proto.TestDep.Foo - - Proto.TestDep.Foo_Fields diff --git a/proto-lens-tests-dep/proto-lens-tests-dep.cabal b/proto-lens-tests-dep/proto-lens-tests-dep.cabal new file mode 100644 index 00000000..985faa50 --- /dev/null +++ b/proto-lens-tests-dep/proto-lens-tests-dep.cabal @@ -0,0 +1,39 @@ +name: proto-lens-tests-dep +version: 0.1.0.1 +synopsis: Test package dependency. +description: This package exports a .proto file which is used by proto-lens-tests to test interpackage dependencies. . NOTE: The test needs a new package with its own .proto files, rather than using the ones in proto-lens-protobuf-types (google/protobuf/*.proto), since on some systems protoc has special logic and can resolve imports of those files automatically. +category: Data +homepage: https://github.com/google/proto-lens#readme +bug-reports: https://github.com/google/proto-lens/issues +author: Judah Jacobson +maintainer: proto-lens@googlegroups.com +copyright: Google Inc. +license: BSD3 +license-file: LICENSE +build-type: Custom +cabal-version: >= 1.24 +extra-source-files: + protos/test-dep/foo.proto + +source-repository head + type: git + location: https://github.com/google/proto-lens + subdir: proto-lens-tests-dep + +custom-setup + setup-depends: + Cabal + , base + , proto-lens-protoc + +library + exposed-modules: + Proto.TestDep.Foo + Proto.TestDep.Foo_Fields + autogen-modules: + Proto.TestDep.Foo + Proto.TestDep.Foo_Fields + build-depends: + base + , proto-lens-protoc + default-language: Haskell2010 diff --git a/release.sh b/release.sh deleted file mode 100755 index 289c7828..00000000 --- a/release.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -ueo pipefail - -cabal sandbox init -export CABAL_SANDBOX_CONFIG=$PWD/cabal.sandbox.config - -for p in lens-labels proto-lens proto-lens-protoc -do - (cd $p && \ - cabal install --force-reinstall) -done - -for p in proto-lens-combinators proto-lens-protobuf-types -do - (cd $p && \ - cabal install --enable-tests --enable-benchmarks --only-dependencies && \ - cabal configure --enable-tests --enable-benchmarks && \ - cabal sdist) -done - diff --git a/travis-cabal.sh b/travis-cabal.sh index 5be2eede..6eb46a7b 100755 --- a/travis-cabal.sh +++ b/travis-cabal.sh @@ -12,7 +12,11 @@ chmod +x $HOME/.local/bin/hpack # List all the packages in this repo. Put certain ones first since # they're dependencies of the others. (Unfortunately, "stack query" doesn't # give them to us in the right order.) -PACKAGES=" + +# All the packages that are prepared to be test-released. +# Don't include test packages, some of which require +# Cabal-2.0 or higher during `sdist` due to using hpack. +PACKAGES_TO_INSTALL=" discrimination-ieee754 lens-labels proto-lens @@ -21,12 +25,17 @@ PACKAGES=" proto-lens-arbitrary proto-lens-combinators proto-lens-optparse + proto-lens-discrimination proto-lens-tests-dep +" + +PACKAGES_TO_BUILD=" proto-lens-tests - proto-lens-discrimination proto-lens-benchmarks -" -echo Building: $PACKAGES +""" + +echo Installing: ${PACKAGES_TO_INSTALL} +echo Building: ${PACKAGES_TO_BUILD} # Needed by haskell-src-exts which is a dependency of proto-lens-protoc. # Sadly, Cabal won't install such build-tools automatically. @@ -35,33 +44,47 @@ cabal install happy # Unregister the already-installed packages, since otherwise they may # propagate between builds. # TODO: use a Cabal sandbox for this. -for p in $PACKAGES +for p in $PACKAGES_TO_INSTALL do echo "Unregistering $p" ghc-pkg unregister --force $p || true done -for p in $PACKAGES +function build_package () { + if [ -f package.yaml ] + # Generate the .cabal file + then hpack + fi + cabal clean + cabal install --enable-tests --force --only-dependencies + cabal configure --enable-tests --enable-benchmarks + cabal build +} + + +for p in $PACKAGES_TO_INSTALL do - echo "Cabal building $p" + echo "Cabal installing $p" (cd $p && - hpack # Generate the .cabal file - cabal clean - cabal install --enable-tests --only-dependencies - cabal configure --enable-tests --enable-benchmarks - cabal build - cabal sdist - SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz"; exit}') - cd dist - if [ -f "$SRC_TGZ" ]; then - cabal install --force-reinstalls "$SRC_TGZ" - else - echo "expected '$SRC_TGZ' not found" - exit 1 - fi + build_package + cabal sdist + SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz"; exit}') + cd dist + if [ -f "$SRC_TGZ" ]; then + cabal install --force-reinstalls "$SRC_TGZ" + else + echo "expected '$SRC_TGZ' not found" + exit 1 + fi ) done +for p in $PACKAGES_TO_BUILD +do + echo "Cabal building $p" + (cd $p && build_package) +done + # Check that "cabal repl" works on a simple test. cd proto-lens-tests # Get rid of the previous dist/autogen to make sure "cabal repl" rebuilds it.